Added First Anal Quest and Double View Casting profile scrapers.
|
@ -171,6 +171,10 @@ module.exports = {
|
||||||
'gloryholesecrets',
|
'gloryholesecrets',
|
||||||
'aziani',
|
'aziani',
|
||||||
'legalporno',
|
'legalporno',
|
||||||
|
[
|
||||||
|
'firstanalquest',
|
||||||
|
'doubleviewcasting',
|
||||||
|
],
|
||||||
[
|
[
|
||||||
'silverstonedvd',
|
'silverstonedvd',
|
||||||
'silviasaint',
|
'silviasaint',
|
||||||
|
|
After Width: | Height: | Size: 15 KiB |
After Width: | Height: | Size: 31 KiB |
After Width: | Height: | Size: 14 KiB |
After Width: | Height: | Size: 15 KiB |
After Width: | Height: | Size: 103 KiB |
After Width: | Height: | Size: 22 KiB |
After Width: | Height: | Size: 96 KiB |
|
@ -564,9 +564,11 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
||||||
// config may group sources to try until success
|
// config may group sources to try until success
|
||||||
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
||||||
try {
|
try {
|
||||||
const scraper = scrapers[scraperSlug];
|
|
||||||
const entity = entitiesBySlug[scraperSlug] || null;
|
const entity = entitiesBySlug[scraperSlug] || null;
|
||||||
|
|
||||||
|
const scraper = scrapers[scraperSlug];
|
||||||
|
const layoutScraper = scraper?.[entity.parameters?.layout] || scraper;
|
||||||
|
|
||||||
const context = {
|
const context = {
|
||||||
...entity,
|
...entity,
|
||||||
// legacy
|
// legacy
|
||||||
|
@ -578,7 +580,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
||||||
|
|
||||||
const label = context.entity?.name;
|
const label = context.entity?.name;
|
||||||
|
|
||||||
if (!scraper?.fetchProfile) {
|
if (!layoutScraper?.fetchProfile) {
|
||||||
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
|
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
|
||||||
throw new Error(`No profile profile scraper available for ${scraperSlug}`);
|
throw new Error(`No profile profile scraper available for ${scraperSlug}`);
|
||||||
}
|
}
|
||||||
|
@ -598,7 +600,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
||||||
|
|
||||||
logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
|
logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
|
||||||
|
|
||||||
const profile = await scraper.fetchProfile(curateActor({
|
const profile = await layoutScraper.fetchProfile(curateActor({
|
||||||
...existingProfile,
|
...existingProfile,
|
||||||
...actor,
|
...actor,
|
||||||
}), context, include);
|
}), context, include);
|
||||||
|
@ -688,7 +690,7 @@ async function scrapeActors(argNames) {
|
||||||
.whereNull('alias_for'),
|
.whereNull('alias_for'),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: entity }), {});
|
const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: acc[entity.slug] || entity }), {});
|
||||||
|
|
||||||
const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({
|
const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({
|
||||||
...acc,
|
...acc,
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const qu = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
|
const slugify = require('../utils/slugify');
|
||||||
|
|
||||||
function scrapeAllA(scenes, channel) {
|
function scrapeAllA(scenes, channel) {
|
||||||
return scenes.map(({ query }) => {
|
return scenes.map(({ query }) => {
|
||||||
|
@ -65,6 +66,53 @@ function scrapeSceneA({ query }, url, channel) {
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function scrapeProfileA({ query, el }, entity) {
|
||||||
|
const profile = {};
|
||||||
|
|
||||||
|
const bio = query.all('.list-model-info li, .profile-info li').reduce((acc, bioEl) => ({
|
||||||
|
...acc,
|
||||||
|
[slugify(query.cnt(bioEl, '.title, span'), '_')]: query.cnt(bioEl, ':nth-child(2)') || query.q(bioEl, ':nth-child(2)', 'title') || query.text(bioEl),
|
||||||
|
}), {});
|
||||||
|
|
||||||
|
profile.dateOfBirth = qu.parseDate(bio.birth_date || bio.date_of_birth, 'DD MMMM, YYYY');
|
||||||
|
profile.birthPlace = bio.nationality || bio.place_of_birth || null;
|
||||||
|
|
||||||
|
profile.weight = Number(bio.weight?.match(/\d+/)?.[0]);
|
||||||
|
profile.height = Number(bio.height?.match(/\d+/)?.[0]);
|
||||||
|
|
||||||
|
profile.eyes = bio.eye_color;
|
||||||
|
profile.hairColor = bio.hair || bio.hair_color;
|
||||||
|
|
||||||
|
profile.aliases = query.text('.sub-title')?.replace(/:\s*/, '').split(/,\s*/);
|
||||||
|
|
||||||
|
if (bio.measurements || bio.body_shape_dimensions) {
|
||||||
|
const [, bust, cup, waist, hip] = (bio.measurements || bio.body_shape_dimensions).match(/(\d+)(\w+)-(\d+)-(\d+)/);
|
||||||
|
|
||||||
|
profile.bust = Number(bust);
|
||||||
|
profile.cup = cup;
|
||||||
|
profile.waist = Number(waist);
|
||||||
|
profile.hip = Number(hip);
|
||||||
|
}
|
||||||
|
|
||||||
|
const description = query.cnt('.model-biography p');
|
||||||
|
const avatar = query.img('.model-box img, .profile-model-photo', 'src', { origin: entity.url });
|
||||||
|
|
||||||
|
if (!/there is no description/.test(description)) {
|
||||||
|
profile.description = description;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (avatar) {
|
||||||
|
profile.avatar = [
|
||||||
|
avatar,
|
||||||
|
avatar.replace('s2_', 's1_'),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
profile.scenes = scrapeAllA(qu.initAll(el, '.list-thumbs .thumb, .main-thumbs > li'), entity);
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchLatestA(channel, page) {
|
async function fetchLatestA(channel, page) {
|
||||||
const url = channel.parameters?.latest
|
const url = channel.parameters?.latest
|
||||||
? `${channel.parameters.latest}/${page}`
|
? `${channel.parameters.latest}/${page}`
|
||||||
|
@ -89,9 +137,33 @@ async function fetchSceneA(url, channel) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function fetchProfileA({ name, slug }, { entity }) {
|
||||||
|
const searchRes = await qu.getAll(`${entity.url}/models/search/?q=${name}`, '.thumb-modal, .big-thumb');
|
||||||
|
|
||||||
|
if (!searchRes.ok) {
|
||||||
|
return searchRes.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
const actor = searchRes.items.find(({ query }) => slugify(query.cnt('.thumb-title a, .title')) === slug);
|
||||||
|
|
||||||
|
if (!actor) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const actorUrl = actor.query.url('a', 'href', { origin: entity.url });
|
||||||
|
const actorRes = await qu.get(actorUrl);
|
||||||
|
|
||||||
|
if (actorRes.ok) {
|
||||||
|
return scrapeProfileA(actorRes.item, entity);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
a: {
|
a: {
|
||||||
fetchLatest: fetchLatestA,
|
fetchLatest: fetchLatestA,
|
||||||
fetchScene: fetchSceneA,
|
fetchScene: fetchSceneA,
|
||||||
|
fetchProfile: fetchProfileA,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
|
@ -198,11 +198,13 @@ const scrapers = {
|
||||||
digitalplayground,
|
digitalplayground,
|
||||||
dtfsluts: fullpornnetwork,
|
dtfsluts: fullpornnetwork,
|
||||||
dorcelclub: dorcel,
|
dorcelclub: dorcel,
|
||||||
|
doubleviewcasting: firstanalquest,
|
||||||
elegantangel,
|
elegantangel,
|
||||||
evilangel,
|
evilangel,
|
||||||
eyeontheguy: hush,
|
eyeontheguy: hush,
|
||||||
fakehub,
|
fakehub,
|
||||||
exploitedcollegegirls: fcuk,
|
exploitedcollegegirls: fcuk,
|
||||||
|
firstanalquest,
|
||||||
forbondage: porndoe,
|
forbondage: porndoe,
|
||||||
freeones,
|
freeones,
|
||||||
gangbangcreampie: aziani,
|
gangbangcreampie: aziani,
|
||||||
|
|