boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Passing context object with site or network instead of scraper slug and 'site or network' to all profile scrapers.

This commit is contained in:
ThePendulum 2020-05-18 03:22:03 +02:00
parent 8733fdc657
commit 885aa4f627
22 changed files with 161 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
config/default.js
View File

@ -91,6 +91,10 @@ module.exports = {
'burningangel',
'brazzers',
'milehighmedia',
[
'devilsfilm',
'roccosiffredi',
],
[
'vixen',
'tushy',
@ -112,6 +116,7 @@ module.exports = {
],
'21sextury',
'julesjordan',
'peternorth',
'naughtyamerica',
'cherrypimps',
'pimpxxx',
@ -143,6 +148,10 @@ module.exports = {
'private',
'ddfnetwork',
'bangbros',
[
'silverstonedvd',
'silviasaint',
],
'kellymadison',
'gangbangcreampie',
'gloryholesecrets',

2
seeds/02_sites.js
View File

@ -1899,6 +1899,7 @@ const sites = [
parameters: {
latest: '/en/All/scenes/0/latest/',
upcoming: '/en/All/scenes/0/upcoming',
classic: true,
},
},
{
@ -1910,6 +1911,7 @@ const sites = [
parameters: {
latest: '/en/scenes/All/0/',
upcoming: '/en/scenes/All/0/1/upcoming',
classic: true,
},
},
{

25
src/actors.js
View File

@ -137,11 +137,10 @@ async function curateProfile(profile) {
name: profile.name,
avatar: profile.avatar,
scraper: profile.scraper,
site: profile.site,
network: profile.network,
};
curatedProfile.site = profile.site.isNetwork ? null : profile.site;
curatedProfile.network = profile.site.isNetwork ? profile.site : null;
curatedProfile.description = profile.description?.trim() || null;
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
curatedProfile.ethnicity = profile.ethnicity?.trim() || null;
@ -288,7 +287,7 @@ async function interpolateProfiles(actors) {
profile.tattoos = getLongest(valuesByProperty.tattoos);
profile.piercings = getLongest(valuesByProperty.piercings);
profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0].id;
profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
return profile;
});
@ -368,21 +367,25 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
try {
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
const scraper = scrapers[scraperSlug];
const siteOrNetwork = networksBySlug[scraperSlug] || sitesBySlug[scraperSlug];
const context = {
site: sitesBySlug[scraperSlug] || null,
network: networksBySlug[scraperSlug] || null,
scraper: scraperSlug,
};
if (!scraper?.fetchProfile) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw new Error(`No profile profile scraper available for ${scraperSlug}`);
}
if (!siteOrNetwork) {
if (!context.site && !context.network) {
logger.warn(`No site or network found for ${scraperSlug}`);
throw new Error(`No site or network found for ${scraperSlug}`);
}
logger.verbose(`Searching profile for '${actor.name}' on '${scraperSlug}'`);
const profile = await scraper.fetchProfile(actor.name, scraperSlug, siteOrNetwork, include);
const profile = await scraper.fetchProfile(actor.name, context, include);
if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure
logger.verbose(`Profile for '${actor.name}' not available on ${scraperSlug}, scraper returned ${profile}`);
@ -392,8 +395,7 @@ async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
return {
...actor,
...profile,
scraper: scraperSlug,
site: siteOrNetwork,
...context,
};
}), Promise.reject(new Error()));
} catch (error) {
@ -424,7 +426,8 @@ async function scrapeActors(actorNames) {
]);
const existingActorEntriesBySlug = existingActorEntries.reduce((acc, actorEntry) => ({ ...acc, [actorEntry.slug]: actorEntry }), {});
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: { ...network, isNetwork: true } }), {});
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: network }), {});
const sitesBySlug = sites.reduce((acc, site) => ({ ...acc, [site.slug]: site }), {});
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlug[baseActor.slug]);
@ -456,6 +459,8 @@ async function scrapeActors(actorNames) {
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actors);
}
return profiles;
}
async function getOrCreateActors(baseActors, batchId) {

8
src/app.js
View File

@ -20,7 +20,13 @@ async function init() {
}
if (argv.actors) {
await scrapeActors(argv.actors);
const actors = await scrapeActors(argv.actors);
if (argv.actorScenes) {
const actorReleases = actors.map(actor => actor.releases).flat().filter(Boolean);
await storeReleases(actorReleases);
}
}
const updateBaseScenes = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates();

4
src/argv.js
View File

@ -32,13 +32,13 @@ const { argv } = yargs
.option('actor-scenes', {
describe: 'Fetch all scenes for an actor',
type: 'boolean',
alias: 'with-releases',
alias: 'with-scenes',
default: false,
})
.option('movie-scenes', {
describe: 'Fetch all scenes for a movie',
type: 'boolean',
alias: 'with-releases',
alias: 'with-scenes',
default: false,
})
.option('scene-movies', {

2
src/scrapers/aziani.js
View File

@ -126,7 +126,7 @@ async function fetchScene(url, site) {
return res.status;
}
async function fetchProfile(actorName, scraperSlug, site) {
async function fetchProfile(actorName, { site }) {
const actorSlug = slugify(actorName, '');
const url = `${site.url}/tour/models/${actorSlug}.html`;
const res = await get(url, '.page-content .row');

2
src/scrapers/bamvisions.js
View File

@ -132,7 +132,7 @@ async function fetchScene(url, site) {
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName, scraperSlug, site, include) {
async function fetchProfile(actorName, { site }, include) {
const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName);

72
src/scrapers/bang.js
View File

@ -80,11 +80,69 @@ function scrapeScene(scene, site) {
return release;
}
function scrapeLatest(scenes, site) {
function scrapeAll(scenes, site) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, site));
}
function scrapeProfile(actor) {
async function fetchActorReleases(actor, site) {
const res = await bhttp.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
query: {
bool: {
must: [
{
match: {
status: 'ok',
},
},
{
nested: {
path: 'actors',
query: {
bool: {
must: [
{
match: {
'actors.mongoId': {
operator: 'AND',
query: actor.id,
},
},
},
],
},
},
},
},
],
must_not: [
{
match: {
type: 'trailer',
},
},
],
},
},
sort: [
{
releaseDate: {
order: 'desc',
},
},
],
}, {
encodeJSON: true,
headers: {
Authorization: `Basic ${authKey}`,
},
});
return scrapeAll(res.body.hits.hits, site);
}
async function scrapeProfile(actor, site, include) {
const profile = {};
profile.aliases = actor.aliases;
@ -115,7 +173,9 @@ function scrapeProfile(actor) {
if (actor.twitter) profile.social = [`https://www.twitter.com/${actor.twitter}`];
if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`;
// TODO: get releases
if (include.releases) {
profile.releases = await fetchActorReleases(actor, site);
}
return profile;
}
@ -204,7 +264,7 @@ async function fetchLatest(site, page = 1) {
},
});
return scrapeLatest(res.body.hits.hits, site);
return scrapeAll(res.body.hits.hits, site);
}
async function fetchScene(url, site) {
@ -220,7 +280,7 @@ async function fetchScene(url, site) {
return scrapeScene(res.body._source, site); // eslint-disable-line no-underscore-dangle
}
async function fetchProfile(actorName) {
async function fetchProfile(actorName, actorSlug, site, include) {
const res = await post(`https://${clusterId}.us-east-1.aws.found.io/actors/actor/_search`, {
size: 5,
sort: [{
@ -255,7 +315,7 @@ async function fetchProfile(actorName) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) {
return scrapeProfile(actor._source);
return scrapeProfile(actor._source, site, include);
}
return null;

4
src/scrapers/blowpass.js
View File

@ -25,8 +25,8 @@ function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.blowpass.com/en/videos/blowpass/latest/All-Categories/0${actorPath}/${page}`;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
async function networkFetchProfile(actorName, context, include) {
return fetchProfile(actorName, context, null, getActorReleasesUrl, include);
}
module.exports = {

2
src/scrapers/brazzers.js
View File

@ -200,7 +200,7 @@ async function fetchScene(url, site) {
return scrapeScene(res.body.toString(), url, site);
}
async function fetchProfile(actorName, scraperSlug, siteOrNetwork, include) {
async function fetchProfile(actorName, context, include) {
const searchUrl = 'https://brazzers.com/pornstars-search/';
const searchRes = await bhttp.get(searchUrl, {
headers: {

10
src/scrapers/cherrypimps.js
View File

@ -120,13 +120,15 @@ async function fetchScene(url, site, release) {
return res.ok ? scrapeScene(res.item, url, site, release) : res.status;
}
async function fetchProfile(actorName, scraperSlug) {
async function fetchProfile(actorName, { site, network, scraper }) {
const actorSlug = slugify(actorName);
const actorSlug2 = slugify(actorName, '');
const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraperSlug)
? [`https://${scraperSlug}.com/models/${actorSlug}.html`, `https://${scraperSlug}.com/models/${actorSlug2}.html`]
: [`https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug}.html`, `https://${scraperSlug.replace('xxx', '')}.xxx/models/${actorSlug2}.html`];
const origin = site?.url || network.url;
const [url, url2] = ['cherrypimps', 'wildoncam'].includes(scraper)
? [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`]
: [`${origin}/models/${actorSlug}.html`, `${origin}/models/${actorSlug2}.html`];
const res = await get(url);
if (res.ok) return scrapeProfile(res.item);

47
src/scrapers/famedigital.js
View File

@ -10,7 +10,7 @@ const {
fetchApiProfile,
scrapeAll,
} = require('./gamma');
const { get } = require('../utils/q');
const { get } = require('../utils/qu');
const slugify = require('../utils/slugify');
function extractLowArtActors(release) {
@ -57,10 +57,19 @@ function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
}
async function fetchClassicProfile(actorName, siteSlug) {
function scrapeClassicProfile({ qu, html }, site) {
const profile = {};
profile.avatar = qu.img('.actorPicture');
profile.releases = scrapeAll(html, null, site.url, false);
return profile;
}
async function fetchClassicProfile(actorName, { site }) {
const actorSlug = slugify(actorName);
const url = `https://${siteSlug}.com/en/pornstars`;
const url = `${site.url}/en/pornstars`;
const pornstarsRes = await get(url);
if (!pornstarsRes.ok) return null;
@ -70,41 +79,23 @@ async function fetchClassicProfile(actorName, siteSlug) {
?.value;
if (actorPath) {
const actorUrl = `https://${siteSlug}.com${actorPath}`;
const actorUrl = `${site.url}${actorPath}`;
const res = await get(actorUrl);
if (res.ok) {
const releases = scrapeAll(res.item, null, `https://www.${siteSlug}.com`, false);
return { releases };
return scrapeClassicProfile(res.item, site);
}
}
return null;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter, silvia] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
include.scenes ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl, include) : [],
include.scenes ? fetchClassicProfile(actorName, 'silviasaint') : [],
include.scenes ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
]);
async function networkFetchProfile(actorName, context, include) {
const profile = await ((context.site.parameters.api && fetchApiProfile(actorName, context, include))
|| (context.site.parameters.classic && include.scenes && fetchClassicProfile(actorName, context, include)) // classic profiles only have scenes, no bio
|| fetchProfile(actorName, context, true, getActorReleasesUrl, include));
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
return {
...peter,
...rocco,
...devils,
releases,
};
}
return null;
return profile;
}
module.exports = {

6
src/scrapers/fullpornnetwork.js
View File

@ -78,12 +78,10 @@ async function fetchScene(url, site) {
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName, scraperSlug) {
async function fetchProfile(actorName, { site }) {
const actorSlug = slugify(actorName, '');
const url = scraperSlug === 'povperverts'
? `https://povperverts.net/models/${actorSlug}.html`
: `https://${scraperSlug}.com/models/${actorSlug}.html`;
const url = `${site.url}/models/${actorSlug}.html`;
const res = await get(url);
return res.ok ? scrapeProfile(res.item, actorName) : res.status;

8
src/scrapers/gamma.js
View File

@ -552,7 +552,9 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
return [];
}
async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesUrl, include) {
async function fetchProfile(actorName, context, altSearchUrl, getActorReleasesUrl, include) {
const siteSlug = context.site?.slug || context.network.slug;
const actorSlug = actorName.toLowerCase().replace(/\s+/, '+');
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
@ -579,7 +581,9 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl, getActorReleasesU
return null;
}
async function fetchApiProfile(actorName, siteSlug, site, include) {
async function fetchApiProfile(actorName, context, include) {
const siteSlug = context.site?.slug || context.network.slug;
const actorSlug = encodeURI(actorName);
const referer = `https://www.${siteSlug}.com/en/search`;

2
src/scrapers/hush.js
View File

@ -381,7 +381,7 @@ async function fetchScene(url, site, baseRelease, beforeFetchLatest) {
return scrapeScene(res.item, site, url, baseRelease);
}
async function fetchProfile(actorName, scraperSlug, site) {
async function fetchProfile(actorName, { site }) {
const actorSlugA = slugify(actorName, '');
const actorSlugB = slugify(actorName);

2
src/scrapers/mikeadriano.js
View File

@ -235,7 +235,7 @@ async function fetchScene(url, site) {
}
/* API protected
async function fetchProfile(actorName, scraperSlug, site) {
async function fetchProfile(actorName, context , site) {
const session = bhttp.session();
await session.get(`https://tour.${site.slug}.com`);

10
src/scrapers/mindgeek.js
View File

@ -222,8 +222,8 @@ async function fetchScene(url, site) {
return null;
}
async function fetchProfile(actorName, networkName, actorPath = 'model') {
const url = `https://www.${networkName}.com`;
async function fetchProfile(actorName, networkSlug, actorPath = 'model') {
const url = `https://www.${networkSlug}.com`;
const { session, instanceToken } = await getSession(url);
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
@ -236,7 +236,7 @@ async function fetchProfile(actorName, networkName, actorPath = 'model') {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) {
const actorUrl = `https://www.${networkName}.com/${actorPath}/${actorData.id}/`;
const actorUrl = `https://www.${networkSlug}.com/${actorPath}/${actorData.id}/`;
const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
const [actorRes, actorReleasesRes] = await Promise.all([
@ -249,11 +249,11 @@ async function fetchProfile(actorName, networkName, actorPath = 'model') {
]);
if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkName);
return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkSlug);
}
if (actorRes.statusCode === 200) {
return scrapeProfile(actorData, actorRes.body.toString(), null, networkName);
return scrapeProfile(actorData, actorRes.body.toString(), null, networkSlug);
}
}
}

4
src/scrapers/nubiles.js
View File

@ -136,9 +136,9 @@ async function fetchScene(url, site) {
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName, siteSlug) {
async function fetchProfile(actorName, { site }) {
const firstLetter = actorName.charAt(0).toLowerCase();
const origin = slugUrlMap[siteSlug] || `https://www.${siteSlug}.com`;
const origin = slugUrlMap[site.slug] || site.url;
const url = `${origin}/model/alpha/${firstLetter}`;
const resModels = await get(url);

6
src/scrapers/score.js
View File

@ -217,7 +217,7 @@ async function fetchScene(url, site) {
return null;
}
async function fetchProfile(actorName, scraperSlug, site, include, page = 1, source = 0) {
async function fetchProfile(actorName, context, include, page = 1, source = 0) {
const letter = actorName.charAt(0).toUpperCase();
const sources = [
@ -244,11 +244,11 @@ async function fetchProfile(actorName, scraperSlug, site, include, page = 1, sou
return null;
}
return fetchProfile(actorName, scraperSlug, site, include, page + 1, source);
return fetchProfile(actorName, context, include, page + 1, source);
}
if (sources[source + 1]) {
return fetchProfile(actorName, scraperSlug, site, include, 1, source + 1);
return fetchProfile(actorName, context, include, 1, source + 1);
}
return null;

7
src/scrapers/scrapers.js
View File

@ -154,12 +154,12 @@ module.exports = {
ddfnetwork,
deeper: vixen,
deeplush: nubiles,
devilsfilm: famedigital,
digitalplayground,
dtfsluts: fullpornnetwork,
evilangel,
eyeontheguy: hush,
fakehub,
famedigital,
freeones,
gangbangcreampie: aziani,
girlfaction: fullpornnetwork,
@ -189,15 +189,19 @@ module.exports = {
nympho: mikeadriano,
onlyprince: fullpornnetwork,
pervertgallery: fullpornnetwork,
peternorth: famedigital,
pimpxxx: cherrypimps,
pornhub,
povperverts: fullpornnetwork,
povpornstars: hush,
private: privateNetwork,
realitykings,
roccosiffredi: famedigital,
score,
seehimfuck: hush,
sexyhub: mindgeek,
silverstonedvd: famedigital,
silviasaint: famedigital,
swallowed: mikeadriano,
thatsitcomshow: nubiles,
transangels,
@ -207,6 +211,7 @@ module.exports = {
twistys,
vixen,
wicked,
wildoncam: cherrypimps,
xempire,
},
};

4
src/scrapers/vixen.js
View File

@ -232,8 +232,8 @@ async function fetchScene(url, site, baseRelease) {
return res.code;
}
async function fetchProfile(actorName, scraperSlug, site, include) {
const origin = `https://www.${scraperSlug}.com`;
async function fetchProfile(actorName, { site }, include) {
const origin = site.url;
const actorSlug = slugify(actorName);
const url = `${origin}/api/${actorSlug}`;
const res = await get(url);

4
src/scrapers/xempire.js
View File

@ -23,8 +23,8 @@ function getActorReleasesUrl(actorPath, page = 1) {
return `https://www.xempire.com/en/videos/xempire/latest/${page}/All-Categories/0${actorPath}`;
}
async function networkFetchProfile(actorName, scraperSlug, site, include) {
return fetchProfile(actorName, scraperSlug, null, getActorReleasesUrl, include);
async function networkFetchProfile(actorName, context, include) {
return fetchProfile(actorName, context, null, getActorReleasesUrl, include);
}
module.exports = {