Added First Anal Quest and Double View Casting profile scrapers.

config/default.js

@@ -171,6 +171,10 @@ module.exports = {
 		'gloryholesecrets',
 		'aziani',
 		'legalporno',
+		[
+			'firstanalquest',
+			'doubleviewcasting',
+		],
 		[
 			'silverstonedvd',
 			'silviasaint',

seeds/04_media.js

@@ -795,7 +795,7 @@ const tagPhotos = [
 	['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
 	['fake-boobs', 14, 'Rikki Six for Dream Dolls'],
 	['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'],
-	['fake-boobs', 17, 'Felina in "With Flors On The Floor" for LouisDeMirabert'],
+	['fake-boobs', 17, 'Felina in "With Flowers On The Floor" for LouisDeMirabert'],
 	['fake-boobs', 18, 'Ebony Godess for Action Girls'],
 	['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
 	['fake-boobs', 16, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],

src/actors.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const config = require('config');
+const util = require('util');
 const Promise = require('bluebird');
 const moment = require('moment');
 const blake2 = require('blake2');
@@ -563,9 +564,11 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 			// config may group sources to try until success
 			return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
 				try {
-					const scraper = scrapers[scraperSlug];
 					const entity = entitiesBySlug[scraperSlug] || null;
 
+					const scraper = scrapers[scraperSlug];
+					const layoutScraper = scraper?.[entity.parameters?.layout] || scraper;
+
 					const context = {
 						...entity,
 						// legacy
@@ -577,7 +580,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 
 					const label = context.entity?.name;
 
-					if (!scraper?.fetchProfile) {
+					if (!layoutScraper?.fetchProfile) {
 						logger.warn(`No profile profile scraper available for ${scraperSlug}`);
 						throw new Error(`No profile profile scraper available for ${scraperSlug}`);
 					}
@@ -597,7 +600,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 
 					logger.verbose(`Searching profile for '${actor.name}' on '${label}'`);
 
-					const profile = await scraper.fetchProfile(curateActor({
+					const profile = await layoutScraper.fetchProfile(curateActor({
 						...existingProfile,
 						...actor,
 					}), context, include);
@@ -687,7 +690,7 @@ async function scrapeActors(argNames) {
 			.whereNull('alias_for'),
 	]);
 
-	const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: entity }), {});
+	const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: acc[entity.slug] || entity }), {});
 
 	const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({
 		...acc,
@@ -731,8 +734,8 @@ async function scrapeActors(argNames) {
 
 	logger.info(`Scraped ${profiles.length} profiles`);
 
-	if (argv.inspect) {
-		console.log(profiles);
+	if (argv.report) {
+		console.log(util.inspect(profiles, { depth: Infinity, colors: true }));
 	}
 
 	if (argv.save) {

src/app.js

@@ -66,7 +66,7 @@ async function init() {
 	const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean) : [];
 	const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
 
-	if (argv.inspect) {
+	if (argv.report) {
 		console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
 		console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
 	}

src/argv.js

@@ -223,7 +223,7 @@ const { argv } = yargs
 		type: 'boolean',
 		default: true,
 	})
-	.option('inspect', {
+	.option('report', {
 		describe: 'Show data in console.',
 		type: 'boolean',
 		default: false,

src/scrapers/firstanalquest.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const qu = require('../utils/qu');
+const slugify = require('../utils/slugify');
 
 function scrapeAllA(scenes, channel) {
 	return scenes.map(({ query }) => {
@@ -29,7 +30,6 @@ function scrapeAllA(scenes, channel) {
 		release.tags = query.cnts('.tags a');
 		release.rating = query.number('.thumb-rating');
 
-		console.log(release);
 		return release;
 	});
 }
@@ -63,10 +63,56 @@ function scrapeSceneA({ query }, url, channel) {
 
 	release.trailer = query.url('a[href*="get_file/"], .download a');
 
-	console.log(release);
 	return release;
 }
 
+function scrapeProfileA({ query, el }, entity) {
+	const profile = {};
+
+	const bio = query.all('.list-model-info li, .profile-info li').reduce((acc, bioEl) => ({
+		...acc,
+		[slugify(query.cnt(bioEl, '.title, span'), '_')]: query.cnt(bioEl, ':nth-child(2)') || query.q(bioEl, ':nth-child(2)', 'title') || query.text(bioEl),
+	}), {});
+
+	profile.dateOfBirth = qu.parseDate(bio.birth_date || bio.date_of_birth, 'DD MMMM, YYYY');
+	profile.birthPlace = bio.nationality || bio.place_of_birth || null;
+
+	profile.weight = Number(bio.weight?.match(/\d+/)?.[0]);
+	profile.height = Number(bio.height?.match(/\d+/)?.[0]);
+
+	profile.eyes = bio.eye_color;
+	profile.hairColor = bio.hair || bio.hair_color;
+
+	profile.aliases = query.text('.sub-title')?.replace(/:\s*/, '').split(/,\s*/);
+
+	if (bio.measurements || bio.body_shape_dimensions) {
+		const [, bust, cup, waist, hip] = (bio.measurements || bio.body_shape_dimensions).match(/(\d+)(\w+)-(\d+)-(\d+)/);
+
+		profile.bust = Number(bust);
+		profile.cup = cup;
+		profile.waist = Number(waist);
+		profile.hip = Number(hip);
+	}
+
+	const description = query.cnt('.model-biography p');
+	const avatar = query.img('.model-box img, .profile-model-photo', 'src', { origin: entity.url });
+
+	if (!/there is no description/.test(description)) {
+		profile.description = description;
+	}
+
+	if (avatar) {
+		profile.avatar = [
+			avatar,
+			avatar.replace('s2_', 's1_'),
+		];
+	}
+
+	profile.scenes = scrapeAllA(qu.initAll(el, '.list-thumbs .thumb, .main-thumbs > li'), entity);
+
+	return profile;
+}
+
 async function fetchLatestA(channel, page) {
 	const url = channel.parameters?.latest
 		? `${channel.parameters.latest}/${page}`
@@ -91,9 +137,33 @@ async function fetchSceneA(url, channel) {
 	return res.status;
 }
 
+async function fetchProfileA({ name, slug }, { entity }) {
+	const searchRes = await qu.getAll(`${entity.url}/models/search/?q=${name}`, '.thumb-modal, .big-thumb');
+
+	if (!searchRes.ok) {
+		return searchRes.status;
+	}
+
+	const actor = searchRes.items.find(({ query }) => slugify(query.cnt('.thumb-title a, .title')) === slug);
+
+	if (!actor) {
+		return null;
+	}
+
+	const actorUrl = actor.query.url('a', 'href', { origin: entity.url });
+	const actorRes = await qu.get(actorUrl);
+
+	if (actorRes.ok) {
+		return scrapeProfileA(actorRes.item, entity);
+	}
+
+	return null;
+}
+
 module.exports = {
 	a: {
 		fetchLatest: fetchLatestA,
 		fetchScene: fetchSceneA,
+		fetchProfile: fetchProfileA,
 	},
 };

src/scrapers/scrapers.js

@@ -198,11 +198,13 @@ const scrapers = {
 		digitalplayground,
 		dtfsluts: fullpornnetwork,
 		dorcelclub: dorcel,
+		doubleviewcasting: firstanalquest,
 		elegantangel,
 		evilangel,
 		eyeontheguy: hush,
 		fakehub,
 		exploitedcollegegirls: fcuk,
+		firstanalquest,
 		forbondage: porndoe,
 		freeones,
 		gangbangcreampie: aziani,