Removed stray slice from Gamma scraper.

src/actors.js

@@ -833,7 +833,7 @@ async function scrapeActors(argNames) {
 	const entitySlugs = sources.flat();
 
 	const [entitiesBySlug, existingActorEntries] = await Promise.all([
-		fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer }),
+		fetchEntitiesBySlug(entitySlugs, { types: ['channel', 'network', 'info'], prefer: argv.prefer || 'channel' }),
 		knex('actors')
 			.select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
 			.whereIn('actors.slug', baseActors.map((baseActor) => baseActor.slug))

src/scrapers/gamma.js

@@ -218,7 +218,7 @@ function curateTitle(title, channel) {
 }
 
 async function scrapeApiReleases(json, site, options) {
-	return json.slice(0, 6).reduce((acc, scene) => {
+	return json.reduce((acc, scene) => {
 		if (options.parameters?.extract && scene.sitename !== options.parameters.extract) {
 			return acc;
 		}
@@ -658,7 +658,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
 	return profile;
 }
 
-async function fetchLatestApi(site, page = 1, options, preData, upcoming = false) {
+async function fetchLatestApi(site, page = 1, options, _preData, upcoming = false) {
 	const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
 	const { apiUrl } = await fetchApiCredentials(referer, site);
 	const slug = options.parameters.querySlug || site.slug;

src/scrapers/porndoe.js

@@ -136,7 +136,7 @@ async function fetchScene(url, channel) {
 }
 
 async function fetchProfile({ name: actorName }, entity, include) {
-	const url = `${entity.url}/models/${slugify(actorName)}.en.html`;
+	const url = `${new URL(entity.url).origin}/models/${slugify(actorName)}.en.html`;
 	const res = await unprint.get(url);
 
 	return res.ok ? scrapeProfile(res.context, url, include) : res.status;

src/scrapers/scrapers.js

@@ -277,19 +277,36 @@ const scrapers = {
 		interracialpovs: hush,
 		povpornstars: hush,
 		seehimfuck: hush,
+		// wankzvr
+		wankzvr,
+		tranzvr: wankzvr,
+		milfvr: wankzvr,
+		// nubilus
+		anilos: nubiles,
+		brattysis: nubiles,
+		deeplush: nubiles,
+		hotcrazymess: nubiles,
+		nfbusty: nubiles,
+		nubilefilms: nubiles,
+		nubiles,
+		thatsitcomshow: nubiles,
+		// porndoe
+		amateureuro: porndoe,
+		forbondage: porndoe,
+		mamacitaz: porndoe,
+		transbella: porndoe,
+		vipsexvault: porndoe,
 		// etc
 		'18vr': badoink,
 		theflourishxxx: theflourish,
 		adultempire,
 		archangel,
 		allherluv: missax,
-		amateureuro: porndoe,
 		americanpornstar,
 		analbbc: fullpornnetwork,
 		analized: fullpornnetwork,
 		analviolation: fullpornnetwork,
 		angelogodshackoriginal,
-		anilos: nubiles,
 		asiam: modelmedia,
 		aziani,
 		'2poles1hole': aziani,
@@ -305,23 +322,19 @@ const scrapers = {
 		vurigvlaanderen: bluedonkeymedia,
 		boobpedia,
 		bradmontana,
-		brattysis: nubiles,
 		cherrypimps,
 		cumlouder,
-		deeplush: nubiles,
 		dorcelclub: dorcel,
 		doubleviewcasting: firstanalquest,
 		dtfsluts: fullpornnetwork,
 		exploitedx, // only from known URL that will specify site
 		firstanalquest,
-		forbondage: porndoe,
 		freeones,
 		girlfaction: fullpornnetwork,
 		hergape: fullpornnetwork,
 		hitzefrei,
 		homemadeanalwhores: fullpornnetwork,
 		hookuphotshot,
-		hotcrazymess: nubiles,
 		inthecrack,
 		jamesdeen: fullpornnetwork,
 		jerkaoke: modelmedia,
@@ -339,18 +352,13 @@ const scrapers = {
 		// analvids,
 		analvids: pornbox,
 		littlecapricedreams,
-		mamacitaz: porndoe,
 		mariskax,
-		milfvr: wankzvr,
 		missax,
 		mylf: teamskeet,
 		mugfucked: fullpornnetwork,
 		naughtyamerica,
 		tonightsgirlfriend: naughtyamerica,
 		nebraskacoeds: elevatedx,
-		nfbusty: nubiles,
-		nubilefilms: nubiles,
-		nubiles,
 		onlyprince: fullpornnetwork,
 		pascalssubsluts,
 		pervcity,
@@ -383,11 +391,7 @@ const scrapers = {
 		teencoreclub,
 		teenmegaworld,
 		testedefudelidade,
-		thatsitcomshow: nubiles,
 		tokyohot,
-		transbella: porndoe,
-		tranzvr: wankzvr,
-		vipsexvault: porndoe,
 		virtualtaboo,
 		darkroomvr: virtualtaboo,
 		onlytarts: virtualtaboo,
@@ -403,7 +407,6 @@ const scrapers = {
 		slayed: vixen,
 		wifey: vixen,
 		vrcosplayx: badoink,
-		wankzvr,
 		wildoncam: cherrypimps,
 	},
 };

src/scrapers/virtualtaboo.js

@@ -105,9 +105,9 @@ function scrapeScene({ query, window }, { url }) {
 function scrapeProfile({ query }) {
 	const profile = {};
 
-	const bioKeys = query.contents('.pornstar-detail__params--top strong, .actor-detail__param-name');
+	const bioKeys = query.contents('.pornstar-detail__params--top strong, .actor-detail__param-name, td.pornstar-detail__info--title');
-	const bioValues = query.exists('.actor-detail__param-value')
+	const bioValues = query.exists('.actor-detail__param-value, .pornstar-detail__info--title')
-		? query.contents('.actor-detail__param-value')
+		? query.contents('.actor-detail__param-value, .pornstar-detail__info--title + td')
 		: query.text('.pornstar-detail__params--top', { join: false })?.map((text) => text.split('•')[0].replace(':', '').trim());
 
 	const bio = Object.fromEntries(bioKeys.map((key, index) => [slugify(key, '_'), bioValues[index]]));

src/scrapers/wankzvr.js

@@ -133,7 +133,7 @@ async function scrapeProfile({ query }, url, entity, options) {
 	profile.description = query.cnt('.person__content');
 
 	profile.gender = entity.slug === 'tranzvr' ? 'transsexual' : 'female';
-	profile.age = bio.age;
+	profile.age = Number(bio.age) || null;
 
 	profile.birthPlace = bio.birthplace;
 

tests/profiles.js

@@ -124,6 +124,27 @@ const actors = [
 	{ entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] },
 	{ entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] },
 	{ entity: 'hookuphotshot', name: 'Kenzie Reeves', fields: ['avatar', 'description'] },
+	{ entity: 'tokyohot', name: 'Mai Kawana', url: 'https://my.tokyo-hot.com/cast/2099/', fields: ['avatar', 'birthPlace', 'height', 'cup', 'bust', 'waist', 'hip', 'hairStyle', 'shoeSize', 'bloodType'] },
+	{ entity: 'teenmegaworld', name: 'Sheri Vi', fields: ['avatar', 'description', 'hairColor', 'eyes'] },
+	// wankz
+	{ entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] },
+	{ entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] },
+	{ entity: 'tranzvr', name: 'Thayssa Fadinha', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'age'] },
+	// virtual taboo
+	{ entity: 'virtualtaboo', name: 'Kama Oxi', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'naturalBoobs', 'hasTattoos'] },
+	{ entity: 'onlytarts', name: 'Kama Oxi', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] },
+	{ entity: 'oopsfamily', name: 'Angel Windell', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] },
+	{ entity: 'darkroomvr', name: 'Alexa Flexy', fields: ['avatar', 'birthPlace', 'dateOfBirth', 'measurements', 'height', 'weight'] },
+	// nubiles
+	{ entity: 'nubiles', name: 'Lolli Pop', fields: ['avatar', 'age', 'description', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'anilos', name: 'Rina Helen', fields: ['avatar', 'age', 'description', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'deeplush', name: 'Penelope Woods', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'hotcrazymess', name: 'Eliza Ibarra', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'nfbusty', name: 'Ella Reese', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'nubilefilms', name: 'Jade Kimiko', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
+	{ entity: 'thatsitcomshow', name: 'Casey Calvert', fields: ['avatar', 'age', 'residencePlace', 'height', 'measurements', 'photos'] },
+	// porndoe
+	{ entity: 'vipsexvault', name: 'Amirah Adara', fields: ['avatar', 'nationality', 'placeOfBirth', 'age', 'naturalBoobs', 'hairColor', 'description'] },
 ];
 
 const actorScrapers = scrapers.actors;
@@ -158,7 +179,7 @@ const validators = {
 	birthCountry: (value) => typeof value === 'string' && value.length > 1,
 	nationality: (value) => typeof value === 'string' && value.length > 3,
 	// height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), // ft in needs to be converted
-	height: (value) => !!Number(value) && value > 150,
+	height: (value) => !!Number(value) && value > 130,
 	weight: (value) => !!Number(value) && value > 40,
 	eyes: (value) => typeof value === 'string' && value.length > 3,
 	hairColor: (value) => typeof value === 'string' && value.length > 3,
@@ -188,7 +209,7 @@ const validators = {
 
 // profiler in this context is shorthand for profile scraper
 async function init() {
-	const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: argv.prefer });
+	const entitiesBySlug = await fetchEntitiesBySlug(Object.keys(actorScrapers), { types: ['channel', 'network', 'info'], prefer: 'channel' });
 
 	Object.entries(actorScrapers).reduce(async (chain, [entitySlug, scraper]) => {
 		await chain;
@@ -218,6 +239,7 @@ async function init() {
 				const profile = await fetchProfile({
 					name: actor.name,
 					slug: slugify(actor.name),
+					url: actor.url,
 				}, {
 					...entity,
 					entity,
@@ -226,13 +248,13 @@ async function init() {
 					parameters: getRecursiveParameters(entity),
 				}, include);
 
-				console.log(profile);
-				console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', '));
-
 				if (!profile) {
 					assert.fail('profile not found');
 				}
 
+				console.log(profile);
+				console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', '));
+
 				await Promise.all(actor.fields.map(async (field) => {
 					assert.ok(
 						validators[field]