Added First Anal Quest and Double View Casting profile scrapers.

This commit is contained in:
DebaucheryLibrarian 2020-12-04 23:53:20 +01:00
parent be1821b9eb
commit 2e0fba3de9
11 changed files with 84 additions and 4 deletions

View File

@ -171,6 +171,10 @@ module.exports = {
'gloryholesecrets', 'gloryholesecrets',
'aziani', 'aziani',
'legalporno', 'legalporno',
[
'firstanalquest',
'doubleviewcasting',
],
[ [
'silverstonedvd', 'silverstonedvd',
'silviasaint', 'silviasaint',

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 103 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 96 KiB

View File

@ -564,9 +564,11 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
// config may group sources to try until success // config may group sources to try until success
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => { return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
try { try {
const scraper = scrapers[scraperSlug];
const entity = entitiesBySlug[scraperSlug] || null; const entity = entitiesBySlug[scraperSlug] || null;
const scraper = scrapers[scraperSlug];
const layoutScraper = scraper?.[entity.parameters?.layout] || scraper;
const context = { const context = {
...entity, ...entity,
// legacy // legacy
@ -578,7 +580,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
const label = context.entity?.name; const label = context.entity?.name;
if (!scraper?.fetchProfile) { if (!layoutScraper?.fetchProfile) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`); logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw new Error(`No profile profile scraper available for ${scraperSlug}`); throw new Error(`No profile profile scraper available for ${scraperSlug}`);
} }
@ -598,7 +600,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
logger.verbose(`Searching profile for '${actor.name}' on '${label}'`); logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
const profile = await scraper.fetchProfile(curateActor({ const profile = await layoutScraper.fetchProfile(curateActor({
...existingProfile, ...existingProfile,
...actor, ...actor,
}), context, include); }), context, include);
@ -688,7 +690,7 @@ async function scrapeActors(argNames) {
.whereNull('alias_for'), .whereNull('alias_for'),
]); ]);
const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: entity }), {}); const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: acc[entity.slug] || entity }), {});
const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({ const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({
...acc, ...acc,

View File

@ -1,6 +1,7 @@
'use strict'; 'use strict';
const qu = require('../utils/qu'); const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
function scrapeAllA(scenes, channel) { function scrapeAllA(scenes, channel) {
return scenes.map(({ query }) => { return scenes.map(({ query }) => {
@ -65,6 +66,53 @@ function scrapeSceneA({ query }, url, channel) {
return release; return release;
} }
function scrapeProfileA({ query, el }, entity) {
const profile = {};
const bio = query.all('.list-model-info li, .profile-info li').reduce((acc, bioEl) => ({
...acc,
[slugify(query.cnt(bioEl, '.title, span'), '_')]: query.cnt(bioEl, ':nth-child(2)') || query.q(bioEl, ':nth-child(2)', 'title') || query.text(bioEl),
}), {});
profile.dateOfBirth = qu.parseDate(bio.birth_date || bio.date_of_birth, 'DD MMMM, YYYY');
profile.birthPlace = bio.nationality || bio.place_of_birth || null;
profile.weight = Number(bio.weight?.match(/\d+/)?.[0]);
profile.height = Number(bio.height?.match(/\d+/)?.[0]);
profile.eyes = bio.eye_color;
profile.hairColor = bio.hair || bio.hair_color;
profile.aliases = query.text('.sub-title')?.replace(/:\s*/, '').split(/,\s*/);
if (bio.measurements || bio.body_shape_dimensions) {
const [, bust, cup, waist, hip] = (bio.measurements || bio.body_shape_dimensions).match(/(\d+)(\w+)-(\d+)-(\d+)/);
profile.bust = Number(bust);
profile.cup = cup;
profile.waist = Number(waist);
profile.hip = Number(hip);
}
const description = query.cnt('.model-biography p');
const avatar = query.img('.model-box img, .profile-model-photo', 'src', { origin: entity.url });
if (!/there is no description/.test(description)) {
profile.description = description;
}
if (avatar) {
profile.avatar = [
avatar,
avatar.replace('s2_', 's1_'),
];
}
profile.scenes = scrapeAllA(qu.initAll(el, '.list-thumbs .thumb, .main-thumbs > li'), entity);
return profile;
}
async function fetchLatestA(channel, page) { async function fetchLatestA(channel, page) {
const url = channel.parameters?.latest const url = channel.parameters?.latest
? `${channel.parameters.latest}/${page}` ? `${channel.parameters.latest}/${page}`
@ -89,9 +137,33 @@ async function fetchSceneA(url, channel) {
return res.status; return res.status;
} }
async function fetchProfileA({ name, slug }, { entity }) {
const searchRes = await qu.getAll(`${entity.url}/models/search/?q=${name}`, '.thumb-modal, .big-thumb');
if (!searchRes.ok) {
return searchRes.status;
}
const actor = searchRes.items.find(({ query }) => slugify(query.cnt('.thumb-title a, .title')) === slug);
if (!actor) {
return null;
}
const actorUrl = actor.query.url('a', 'href', { origin: entity.url });
const actorRes = await qu.get(actorUrl);
if (actorRes.ok) {
return scrapeProfileA(actorRes.item, entity);
}
return null;
}
module.exports = { module.exports = {
a: { a: {
fetchLatest: fetchLatestA, fetchLatest: fetchLatestA,
fetchScene: fetchSceneA, fetchScene: fetchSceneA,
fetchProfile: fetchProfileA,
}, },
}; };

View File

@ -198,11 +198,13 @@ const scrapers = {
digitalplayground, digitalplayground,
dtfsluts: fullpornnetwork, dtfsluts: fullpornnetwork,
dorcelclub: dorcel, dorcelclub: dorcel,
doubleviewcasting: firstanalquest,
elegantangel, elegantangel,
evilangel, evilangel,
eyeontheguy: hush, eyeontheguy: hush,
fakehub, fakehub,
exploitedcollegegirls: fcuk, exploitedcollegegirls: fcuk,
firstanalquest,
forbondage: porndoe, forbondage: porndoe,
freeones, freeones,
gangbangcreampie: aziani, gangbangcreampie: aziani,