Updated profile scrapers to use base actor instead of actor name. Fixes for Reality Kings and Cherry Pimps scrapers.
This commit is contained in:
parent
939eba8e61
commit
dff4d15872
|
@ -610,7 +610,7 @@ async function scrapeActors(actorNames) {
|
||||||
},
|
},
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlugAndEntryId[baseActor.slug][baseActor.entryId]);
|
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlugAndEntryId[baseActor.slug]?.[baseActor.entryId]);
|
||||||
|
|
||||||
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
|
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
|
||||||
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
|
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
|
||||||
|
|
|
@ -126,7 +126,7 @@ async function fetchScene(url, site) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }) {
|
async function fetchProfile({ name: actorName }, { site }) {
|
||||||
const actorSlug = slugify(actorName, '');
|
const actorSlug = slugify(actorName, '');
|
||||||
const url = `${site.url}/tour/models/${actorSlug}.html`;
|
const url = `${site.url}/tour/models/${actorSlug}.html`;
|
||||||
const res = await get(url, '.page-content .row');
|
const res = await get(url, '.page-content .row');
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'babes');
|
return fetchProfile({ name: actorName }, 'babes');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -132,7 +132,7 @@ async function fetchScene(url, site) {
|
||||||
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }, include) {
|
async function fetchProfile({ name: actorName }, { site }, include) {
|
||||||
const actorSlugA = slugify(actorName, '');
|
const actorSlugA = slugify(actorName, '');
|
||||||
const actorSlugB = slugify(actorName);
|
const actorSlugB = slugify(actorName);
|
||||||
|
|
||||||
|
|
|
@ -278,7 +278,7 @@ async function fetchScene(url) {
|
||||||
return scrapeScene(res.body._source); // eslint-disable-line no-underscore-dangle
|
return scrapeScene(res.body._source); // eslint-disable-line no-underscore-dangle
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, context, include) {
|
async function fetchProfile({ name: actorName }, context, include) {
|
||||||
const res = await post(`https://${clusterId}.us-east-1.aws.found.io/actors/actor/_search`, {
|
const res = await post(`https://${clusterId}.us-east-1.aws.found.io/actors/actor/_search`, {
|
||||||
size: 5,
|
size: 5,
|
||||||
sort: [{
|
sort: [{
|
||||||
|
|
|
@ -221,7 +221,7 @@ async function fetchScene(url, site, release) {
|
||||||
return scrapeScene(res.item.html, url, site);
|
return scrapeScene(res.item.html, url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, scope) {
|
async function fetchProfile({ name: actorName }, scope) {
|
||||||
const actorSlug = slugify(actorName);
|
const actorSlug = slugify(actorName);
|
||||||
const url = `https://bangbros.com/search/${actorSlug}`;
|
const url = `https://bangbros.com/search/${actorSlug}`;
|
||||||
const res = await bhttp.get(url);
|
const res = await bhttp.get(url);
|
||||||
|
|
|
@ -25,8 +25,8 @@ function getActorReleasesUrl(actorPath, page = 1) {
|
||||||
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
|
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function networkFetchProfile(actorName, context, include) {
|
async function networkFetchProfile({ name: actorName }, context, include) {
|
||||||
return fetchProfile(actorName, context, null, getActorReleasesUrl, include);
|
return fetchProfile({ name: actorName }, context, null, getActorReleasesUrl, include);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -78,7 +78,7 @@ function scrapeProfile(html) {
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlug = actorName.replace(/\s+/, '_');
|
const actorSlug = actorName.replace(/\s+/, '_');
|
||||||
const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
|
const res = await bhttp.get(`http://www.boobpedia.com/boobs/${actorSlug}`);
|
||||||
|
|
||||||
|
|
|
@ -185,7 +185,7 @@ async function fetchScene(url, site) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, context, include) {
|
async function fetchProfile({ name: actorName }, context, include) {
|
||||||
const searchRes = await qu.get('https://brazzers.com/pornstars-search/', `a[title="${actorName}" i]`, {
|
const searchRes = await qu.get('https://brazzers.com/pornstars-search/', `a[title="${actorName}" i]`, {
|
||||||
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
|
Cookie: `textSearch=${encodeURIComponent(actorName)};`,
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,31 +1,31 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { get, geta, ctxa, ed } = require('../utils/q');
|
const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
|
||||||
function scrapeAll(scenes, site) {
|
function scrapeAll(scenes, site) {
|
||||||
return scenes.map(({ qu }) => {
|
return scenes.map(({ query }) => {
|
||||||
const url = qu.url('.text-thumb a');
|
const url = query.url('.text-thumb a');
|
||||||
const { pathname } = new URL(url);
|
const { pathname } = new URL(url);
|
||||||
const channelUrl = qu.url('.badge');
|
const channelUrl = query.url('.badge');
|
||||||
|
|
||||||
if (site?.parameters?.extract && qu.q('.badge', true) !== site.name) {
|
if (site?.parameters?.extract && query.q('.badge', true) !== site.name) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
release.url = channelUrl ? `${channelUrl}${pathname}` : url;
|
release.url = channelUrl ? `${channelUrl}${pathname}` : url;
|
||||||
release.entryId = pathname.match(/\/\d+/)[0].slice(1);
|
release.entryId = pathname.match(/\/trailers\/(.*).html/)[1];
|
||||||
release.title = qu.q('.text-thumb a', true);
|
release.title = query.q('.text-thumb a', true);
|
||||||
|
|
||||||
release.date = qu.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/);
|
release.date = query.date('.date', 'YYYY-MM-DD', /\d{4}-\d{2}-\d{2}/);
|
||||||
release.duration = qu.dur('.date', /(\d{2}:)?\d{2}:\d{2}/);
|
release.duration = query.dur('.date', /(\d{2}:)?\d{2}:\d{2}/);
|
||||||
|
|
||||||
release.actors = qu.all('.category a', true);
|
release.actors = query.all('.category a', true);
|
||||||
|
|
||||||
release.poster = qu.img('img.video_placeholder, .video-images img');
|
release.poster = query.img('img.video_placeholder, .video-images img');
|
||||||
release.teaser = { src: qu.trailer() };
|
release.teaser = { src: query.trailer() };
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}).filter(Boolean);
|
}).filter(Boolean);
|
||||||
|
@ -56,18 +56,18 @@ function scrapeScene({ q, qd, qa }, url, _site, baseRelease) {
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeProfile({ q, qa, qtx }) {
|
function scrapeProfile({ query }) {
|
||||||
const profile = {};
|
const profile = {};
|
||||||
|
|
||||||
const keys = qa('.model-descr_line:not(.model-descr_rait) p.text span', true);
|
const keys = query.all('.model-descr_line:not(.model-descr_rait) p.text span', true);
|
||||||
const values = qa('.model-descr_line:not(.model-descr_rait) p.text').map(el => qtx(el));
|
const values = query.all('.model-descr_line:not(.model-descr_rait) p.text').map(el => query.text(el));
|
||||||
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
|
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, '_')]: values[index] }), {});
|
||||||
|
|
||||||
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)cm\)/)[1]);
|
if (bio.height) profile.height = Number(bio.height.match(/\((\d+)\s*cm\)/)?.[1]);
|
||||||
if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)[1]);
|
if (bio.weight) profile.weight = Number(bio.weight.match(/\((\d+)kg\)/)?.[1]);
|
||||||
if (bio.race) profile.ethnicity = bio.race;
|
if (bio.race) profile.ethnicity = bio.race;
|
||||||
|
|
||||||
if (bio.date_of_birth) profile.birthdate = ed(bio.date_of_birth, 'MMMM D, YYYY');
|
if (bio.date_of_birth) profile.birthdate = qu.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
|
||||||
if (bio.birthplace) profile.birthPlace = bio.birthplace;
|
if (bio.birthplace) profile.birthPlace = bio.birthplace;
|
||||||
|
|
||||||
if (bio.measurements) {
|
if (bio.measurements) {
|
||||||
|
@ -96,11 +96,11 @@ function scrapeProfile({ q, qa, qtx }) {
|
||||||
|
|
||||||
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
|
if (bio.aliases) profile.aliases = bio.aliases.split(',').map(alias => alias.trim());
|
||||||
|
|
||||||
const avatar = q('.model-img img');
|
const avatar = query.q('.model-img img');
|
||||||
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
|
profile.avatar = avatar.getAttribute('src0_3x') || avatar.getAttribute('src0_2x') || avatar.dataset.src;
|
||||||
|
|
||||||
const releases = qa('.video-thumb');
|
const releases = query.all('.video-thumb');
|
||||||
profile.releases = scrapeAll(ctxa(releases));
|
profile.releases = scrapeAll(qu.initAll(releases));
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
@ -109,18 +109,18 @@ async function fetchLatest(site, page = 1) {
|
||||||
const url = site.parameters?.extract
|
const url = site.parameters?.extract
|
||||||
? `https://cherrypimps.com/categories/movies_${page}.html`
|
? `https://cherrypimps.com/categories/movies_${page}.html`
|
||||||
: `${site.url}/categories/movies_${page}.html`;
|
: `${site.url}/categories/movies_${page}.html`;
|
||||||
const res = await geta(url, 'div.video-thumb');
|
const res = await qu.getAll(url, 'div.video-thumb');
|
||||||
|
|
||||||
return res.ok ? scrapeAll(res.items, site) : res.status;
|
return res.ok ? scrapeAll(res.items, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchScene(url, site, release) {
|
async function fetchScene(url, site, release) {
|
||||||
const res = await get(url);
|
const res = await qu.get(url);
|
||||||
|
|
||||||
return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
|
return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site, network, scraper }) {
|
async function fetchProfile({ name: actorName }, { site, network, scraper }) {
|
||||||
const actorSlug = slugify(actorName);
|
const actorSlug = slugify(actorName);
|
||||||
const actorSlug2 = slugify(actorName, '');
|
const actorSlug2 = slugify(actorName, '');
|
||||||
|
|
||||||
|
@ -130,10 +130,10 @@ async function fetchProfile(actorName, { site, network, scraper }) {
|
||||||
? [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`]
|
? [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`]
|
||||||
: [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`];
|
: [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`];
|
||||||
|
|
||||||
const res = await get(url);
|
const res = await qu.get(url);
|
||||||
if (res.ok) return scrapeProfile(res.item);
|
if (res.ok) return scrapeProfile(res.item);
|
||||||
|
|
||||||
const res2 = await get(url2);
|
const res2 = await qu.get(url2);
|
||||||
return res2.ok ? scrapeProfile(res2.item) : res2.status;
|
return res2.ok ? scrapeProfile(res2.item) : res2.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -156,7 +156,7 @@ async function fetchScene(url, site) {
|
||||||
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
|
const resSearch = await bhttp.post('https://ddfnetwork.com/search/ajax',
|
||||||
{
|
{
|
||||||
type: 'hints',
|
type: 'hints',
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'digitalplayground', 'modelprofile');
|
return fetchProfile({ name: actorName }, 'digitalplayground', 'modelprofile');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'fakehub', 'modelprofile');
|
return fetchProfile({ name: actorName }, 'fakehub', 'modelprofile');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -90,10 +90,10 @@ async function fetchClassicProfile(actorName, { site }) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function networkFetchProfile(actorName, context, include) {
|
async function networkFetchProfile({ name: actorName }, context, include) {
|
||||||
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|
const profile = await ((context.site.parameters?.api && fetchApiProfile(actorName, context, include))
|
||||||
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|
|| (context.site.parameters?.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|
||||||
|| fetchProfile(actorName, context, true, getActorReleasesUrl, include));
|
|| fetchProfile({ name: actorName }, context, true, getActorReleasesUrl, include));
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,7 +63,7 @@ function scrapeSearch(html) {
|
||||||
return document.querySelector('a.image-link')?.href || null;
|
return document.querySelector('a.image-link')?.href || null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
|
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
|
||||||
|
|
||||||
const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`);
|
const res = await bhttp.get(`https://freeones.nl/${actorSlug}/profile`);
|
||||||
|
|
|
@ -107,7 +107,7 @@ async function scrapeProfileBio(html, frontpageProfile, url, name) {
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const slug = actorName.replace(' ', '_');
|
const slug = actorName.replace(' ', '_');
|
||||||
const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`;
|
const frontpageUrl = `https://www.freeones.com/html/v_links/${slug}`;
|
||||||
|
|
||||||
|
|
|
@ -96,7 +96,7 @@ async function fetchScene(url, site) {
|
||||||
return res.ok && res.item ? scrapeScene(res.item, url, site) : res.status;
|
return res.ok && res.item ? scrapeScene(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }) {
|
async function fetchProfile({ name: actorName }, { site }) {
|
||||||
const actorSlug = slugify(actorName, '');
|
const actorSlug = slugify(actorName, '');
|
||||||
|
|
||||||
const url = `${site.url}/1/model/${actorSlug}`;
|
const url = `${site.url}/1/model/${actorSlug}`;
|
||||||
|
|
|
@ -559,7 +559,7 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, context, altSearchUrl, getActorReleasesUrl, include) {
|
async function fetchProfile({ name: actorName }, context, altSearchUrl, getActorReleasesUrl, include) {
|
||||||
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
|
const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
|
||||||
|
|
||||||
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
|
||||||
|
|
|
@ -381,7 +381,7 @@ async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
|
||||||
return scrapeScene(res.item, site, url, baseRelease);
|
return scrapeScene(res.item, site, url, baseRelease);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }) {
|
async function fetchProfile({ name: actorName }, { site }) {
|
||||||
const actorSlugA = slugify(actorName, '');
|
const actorSlugA = slugify(actorName, '');
|
||||||
const actorSlugB = slugify(actorName);
|
const actorSlugB = slugify(actorName);
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchProfile } = require('./mindgeek');
|
const { fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'iconmale');
|
return fetchProfile({ name: actorName }, 'iconmale');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -381,7 +381,7 @@ async function fetchMovie(url, site) {
|
||||||
return res.ok ? scrapeMovie(res.item, url, site) : res.status;
|
return res.ok ? scrapeMovie(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlugA = slugify(actorName, '-');
|
const actorSlugA = slugify(actorName, '-');
|
||||||
const actorSlugB = slugify(actorName, '');
|
const actorSlugB = slugify(actorName, '');
|
||||||
|
|
||||||
|
|
|
@ -154,7 +154,7 @@ async function fetchScene(url, channel, baseRelease) {
|
||||||
return res.ok ? scrapeScene(res.item, url, baseRelease) : res.status;
|
return res.ok ? scrapeScene(res.item, url, baseRelease) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlug = slugify(actorName);
|
const actorSlug = slugify(actorName);
|
||||||
const res = await qu.get(`https://www.kellymadison.com/models/${actorSlug}`, null, {
|
const res = await qu.get(`https://www.kellymadison.com/models/${actorSlug}`, null, {
|
||||||
'X-Requested-With': 'XMLHttpRequest',
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
|
|
@ -103,7 +103,7 @@ async function fetchScene(url, channel) {
|
||||||
return res.ok ? scrapeScene(res.item, url, channel) : res.status;
|
return res.ok ? scrapeScene(res.item, url, channel) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, entity, include) {
|
async function fetchProfile({ name: actorName }, entity, include) {
|
||||||
const url = `http://killergram.com/episodes.asp?page=episodes&model=${encodeURI(actorName)}&ct=model`;
|
const url = `http://killergram.com/episodes.asp?page=episodes&model=${encodeURI(actorName)}&ct=model`;
|
||||||
const res = await qu.get(url, '#content', null, {
|
const res = await qu.get(url, '#content', null, {
|
||||||
followRedirects: false,
|
followRedirects: false,
|
||||||
|
|
|
@ -148,7 +148,7 @@ async function fetchScene(url, site) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, entity, include) {
|
async function fetchProfile({ name: actorName }, entity, include) {
|
||||||
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
|
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
|
||||||
|
|
||||||
if (searchRes.ok) {
|
if (searchRes.ok) {
|
||||||
|
|
|
@ -179,7 +179,7 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site, useGallery);
|
return scrapeScene(res.body.toString(), url, site, useGallery);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const res = await bhttp.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
|
const res = await bhttp.get(`https://www.legalporno.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
|
||||||
const data = res.body;
|
const data = res.body;
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'men', 'modelprofile');
|
return fetchProfile({ name: actorName }, 'men', 'modelprofile');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'devianthardcore');
|
return fetchProfile({ name: actorName }, 'devianthardcore');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -86,7 +86,7 @@ async function fetchScene(url, channel) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* API protected
|
/* API protected
|
||||||
async function fetchProfile(actorName, context , site) {
|
async function fetchProfile({ name: actorName }, context , site) {
|
||||||
const session = bhttp.session();
|
const session = bhttp.session();
|
||||||
|
|
||||||
await session.get(`https://tour.${site.slug}.com`);
|
await session.get(`https://tour.${site.slug}.com`);
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'milehighmedia');
|
return fetchProfile({ name: actorName }, 'milehighmedia');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
|
@ -7,7 +6,7 @@ const bhttp = require('bhttp');
|
||||||
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
const { ex } = require('../utils/q');
|
const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const { inchesToCm, lbsToKg } = require('../utils/convert');
|
const { inchesToCm, lbsToKg } = require('../utils/convert');
|
||||||
const { cookieToData } = require('../utils/cookies');
|
const { cookieToData } = require('../utils/cookies');
|
||||||
|
@ -140,7 +139,7 @@ async function getSession(url) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeProfile(data, html, releases = [], networkName) {
|
function scrapeProfile(data, html, releases = [], networkName) {
|
||||||
const { qa, qd } = ex(html);
|
const { query } = qu.extract(html);
|
||||||
|
|
||||||
const profile = {
|
const profile = {
|
||||||
description: data.bio,
|
description: data.bio,
|
||||||
|
@ -169,8 +168,8 @@ function scrapeProfile(data, html, releases = [], networkName) {
|
||||||
|| data.images.card_main_rect[0].xs?.url;
|
|| data.images.card_main_rect[0].xs?.url;
|
||||||
}
|
}
|
||||||
|
|
||||||
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
|
const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent));
|
||||||
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
|
if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
|
||||||
|
|
||||||
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
|
profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));
|
||||||
|
|
||||||
|
@ -222,7 +221,7 @@ async function fetchScene(url, site) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, networkSlug, actorPath = 'model') {
|
async function fetchProfile({ name: actorName }, networkSlug, actorPath = 'model') {
|
||||||
const url = `https://www.${networkSlug}.com`;
|
const url = `https://www.${networkSlug}.com`;
|
||||||
const { session, instanceToken } = await getSession(url);
|
const { session, instanceToken } = await getSession(url);
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'mofos');
|
return fetchProfile({ name: actorName }, 'mofos');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -139,7 +139,7 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlug = slugify(actorName);
|
const actorSlug = slugify(actorName);
|
||||||
|
|
||||||
const res = await bhttp.get(`https://www.naughtyamerica.com/pornstar/${actorSlug}`);
|
const res = await bhttp.get(`https://www.naughtyamerica.com/pornstar/${actorSlug}`);
|
||||||
|
|
|
@ -136,7 +136,7 @@ async function fetchScene(url, site) {
|
||||||
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }) {
|
async function fetchProfile({ name: actorName }, { site }) {
|
||||||
const firstLetter = actorName.charAt(0).toLowerCase();
|
const firstLetter = actorName.charAt(0).toLowerCase();
|
||||||
const origin = slugUrlMap[site.slug] || site.url;
|
const origin = slugUrlMap[site.slug] || site.url;
|
||||||
|
|
||||||
|
|
|
@ -100,7 +100,7 @@ async function fetchScene(url, entity) {
|
||||||
return res.ok ? scrapeScene(res.item, entity) : res.status;
|
return res.ok ? scrapeScene(res.item, entity) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const url = `https://pervcity.com/models/${slugify(actorName)}.html`;
|
const url = `https://pervcity.com/models/${slugify(actorName)}.html`;
|
||||||
const res = await qu.get(url);
|
const res = await qu.get(url);
|
||||||
|
|
||||||
|
|
|
@ -100,8 +100,6 @@ async function scrapeProfile({ query }, url, include) {
|
||||||
profile.releases = await fetchActorReleases({ query }, url);
|
profile.releases = await fetchActorReleases({ query }, url);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(profile);
|
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -117,7 +115,7 @@ async function fetchScene(url, channel) {
|
||||||
return res.ok ? scrapeScene(res.item, url, channel) : res.status;
|
return res.ok ? scrapeScene(res.item, url, channel) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, entity, include) {
|
async function fetchProfile({ name: actorName }, entity, include) {
|
||||||
const url = `http://letsdoeit.com/models/${slugify(actorName)}.en.html`;
|
const url = `http://letsdoeit.com/models/${slugify(actorName)}.en.html`;
|
||||||
const res = await qu.get(url);
|
const res = await qu.get(url);
|
||||||
|
|
||||||
|
|
|
@ -51,7 +51,7 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
|
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
|
||||||
|
|
||||||
/* Model pages are not reliably associated with actual porn stars
|
/* Model pages are not reliably associated with actual porn stars
|
||||||
|
|
|
@ -175,7 +175,7 @@ async function fetchScene(url, site) {
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile({ name: actorName }) {
|
||||||
const actorSearchSlug = slugify(actorName, '+');
|
const actorSearchSlug = slugify(actorName, '+');
|
||||||
const url = `https://www.private.com/search.php?query=${actorSearchSlug}`;
|
const url = `https://www.private.com/search.php?query=${actorSearchSlug}`;
|
||||||
const modelRes = await geta(url, '.model h3 a');
|
const modelRes = await geta(url, '.model h3 a');
|
||||||
|
|
|
@ -41,8 +41,8 @@ async function fetchLatestWrap(site, page = 1) {
|
||||||
return fetchLatest(site, page);
|
return fetchLatest(site, page);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'realitykings');
|
return fetchProfile({ name: actorName }, 'realitykings');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -217,7 +217,7 @@ async function fetchScene(url, site) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, context, include, page = 1, source = 0) {
|
async function fetchProfile({ name: actorName }, context, include, page = 1, source = 0) {
|
||||||
const letter = actorName.charAt(0).toUpperCase();
|
const letter = actorName.charAt(0).toUpperCase();
|
||||||
|
|
||||||
const sources = [
|
const sources = [
|
||||||
|
@ -244,11 +244,11 @@ async function fetchProfile(actorName, context, include, page = 1, source = 0) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return fetchProfile(actorName, context, include, page + 1, source);
|
return fetchProfile({ name: actorName }, context, include, page + 1, source);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sources[source + 1]) {
|
if (sources[source + 1]) {
|
||||||
return fetchProfile(actorName, context, include, 1, source + 1);
|
return fetchProfile({ name: actorName }, context, include, 1, source + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -134,7 +134,7 @@ async function fetchScene(url, entity) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { entity }, include) {
|
async function fetchProfile({ name: actorName }, { entity }, include) {
|
||||||
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
|
const res = await http.get(`https://teencoreclub.com/api/actors?query=${actorName}`);
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
|
|
|
@ -79,7 +79,7 @@ async function fetchScene(url, channel) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, entity, include) {
|
async function fetchProfile({ name: actorName }, entity, include) {
|
||||||
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
|
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
|
||||||
const res = await qu.get(url);
|
const res = await qu.get(url);
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchProfile } = require('./mindgeek');
|
const { fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'transangels');
|
return fetchProfile({ name: actorName }, 'transangels');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
const { fetchScene, fetchLatest, fetchProfile } = require('./mindgeek');
|
||||||
|
|
||||||
async function networkFetchProfile(actorName) {
|
async function networkFetchProfile({ name: actorName }) {
|
||||||
return fetchProfile(actorName, 'twistys');
|
return fetchProfile({ name: actorName }, 'twistys');
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -233,7 +233,7 @@ async function fetchScene(url, site, baseRelease) {
|
||||||
return res.code;
|
return res.code;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName, { site }, include) {
|
async function fetchProfile({ name: actorName }, { site }, include) {
|
||||||
const origin = site.url;
|
const origin = site.url;
|
||||||
const actorSlug = slugify(actorName);
|
const actorSlug = slugify(actorName);
|
||||||
const url = `${origin}/api/${actorSlug}`;
|
const url = `${origin}/api/${actorSlug}`;
|
||||||
|
|
|
@ -23,8 +23,8 @@ function getActorReleasesUrl(actorPath, page = 1) {
|
||||||
return `https://www.xempire.com/en/videos/xempire/latest/${page}/All-Categories/0${actorPath}`;
|
return `https://www.xempire.com/en/videos/xempire/latest/${page}/All-Categories/0${actorPath}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function networkFetchProfile(actorName, context, include) {
|
async function networkFetchProfile(baseActor, context, include) {
|
||||||
return fetchProfile(actorName, context, null, getActorReleasesUrl, include);
|
return fetchProfile(baseActor, context, null, getActorReleasesUrl, include);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -326,7 +326,7 @@ function init(element, window) {
|
||||||
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
|
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
|
||||||
.reduce((acc, [key, func]) => ({
|
.reduce((acc, [key, func]) => ({
|
||||||
...acc,
|
...acc,
|
||||||
[key]: (...args) => (args[0].nodeType === undefined // allow for different context
|
[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
|
||||||
? func(element, ...args)
|
? func(element, ...args)
|
||||||
: func(...args)),
|
: func(...args)),
|
||||||
}), {});
|
}), {});
|
||||||
|
|
Loading…
Reference in New Issue