Added Aylo and MindGeek profile tests, updated site configuration.

seeds/01_networks.js

@@ -358,6 +358,7 @@ const networks = [
 		description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
 		parameters: {
 			mobile: 'https://m.dpfanatics.com/en/video',
+			layout: 'api',
 		},
 		parent: 'gamma',
 	},
@@ -794,6 +795,9 @@ const networks = [
 		url: 'https://www.vivid.com',
 		description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
 		parent: 'gamma',
+		parameters: {
+			layout: 'api',
+		},
 	},
 	{
 		slug: 'vixen',
@@ -827,7 +831,7 @@ const networks = [
 		slug: 'zerotolerance',
 		name: 'Zero Tolerance',
 		alias: ['ztod'],
-		url: 'http://www.ztod.com',
+		url: 'https://www.zerotolerancefilms.com',
 		parameters: {
 			layout: 'api',
 		},

seeds/02_sites.js

@@ -568,18 +568,6 @@ const sites = [
 	},
 	*/
 	// ADULT TIME INDEPENDENT
-	{
-		name: 'Blake Mason',
-		slug: 'blakemason',
-		url: 'https://www.blakemason.com',
-		parent: 'gamma',
-		tags: ['gay'],
-		independent: true,
-		parameters: {
-			layout: 'api',
-			referer: 'https://freetour.adulttime.com/en/join',
-		},
-	},
 	{
 		name: 'ChaosMen',
 		slug: 'chaosmen',
@@ -975,6 +963,7 @@ const sites = [
 		independent: true,
 		parameters: {
 			scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
+			layout: 'api',
 		},
 	},
 	{
@@ -986,6 +975,7 @@ const sites = [
 		independent: true,
 		parameters: {
 			scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
+			layout: 'api',
 		},
 	},
 	/* different layout
@@ -998,6 +988,7 @@ const sites = [
 		independent: true,
 		parameters: {
 			scene: 'https://www.portagloryhole.com/scenes',
+			layout: 'api',
 		},
 	},
 	*/
@@ -3949,7 +3940,6 @@ const sites = [
 		name: 'Devil\'s Film',
 		url: 'https://www.devilsfilm.com',
 		description: 'Welcome to the best porn network, DevilsFilm.com, featuring teens, MILFs, trans and interracial porn with all of your favorite pornstars in 4k ultra HD!',
-		parameters: { api: true },
 		parent: 'famedigital',
 	},
 	{
@@ -3969,7 +3959,10 @@ const sites = [
 		url: 'https://www.daringsexhd.com/',
 		description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
 		parent: 'famedigital',
-		parameters: { api: true },
+		parameters: {
+			api: true,
+			layout: null,
+		},
 	},
 	{
 		slug: 'peternorth',
@@ -3987,7 +3980,6 @@ const sites = [
 		name: 'Rocco Siffredi',
 		url: 'https://www.roccosiffredi.com',
 		description: 'Welcome to the official RoccoSiffredi.com, the Italian Stallion, with hardcore anal fucking and rough sex from the man himself who has coined the term hardcore.',
-		parameters: { api: true },
 		parent: 'famedigital',
 	},
 	{
@@ -4994,6 +4986,14 @@ const sites = [
 		name: 'InTheCrack',
 		url: 'https://inthecrack.com',
 	},
+	// TODO: INDIE https://nats.indiebucks.com/external.php?page=sites
+	{
+		// used to be Gamma
+		name: 'Blake Mason',
+		slug: 'blakemason',
+		url: 'https://www.blakemason.com',
+		tags: ['gay'],
+	},
 	// INN OF SIN
 	{
 		slug: 'richardmannevents',

src/scrapers/gamma.js

@@ -624,7 +624,7 @@ async function scrapeProfile({ query }, url, actorName, _siteSlug, getActorRelea
 
 	profile.description = query.cnt('.actorBio p:not(.bioTitle)');
 
-	if (hair) profile.hair = hair.split(':')[1].trim();
+	if (hair) profile.hairColor = hair.split(':')[1].trim();
 	if (height) profile.height = Number(height.match(/\d+/)[0]);
 	if (weight) profile.weight = Number(weight.match(/\d+/)[0]);
 	if (alias) profile.aliases = alias.split(':')[1].trim().split(', ');
@@ -648,7 +648,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
 
 	if (data.attributes.ethnicity) profile.ethnicity = data.attributes.ethnicity;
 	if (data.attributes.eye_color) profile.eyes = data.attributes.eye_color;
-	if (data.attributes.hair_color) profile.hair = data.attributes.hair_color;
+	if (data.attributes.hair_color) profile.hairColor = data.attributes.hair_color;
 
 	const avatarPaths = Object.values(data.pictures).reverse();
 	if (avatarPaths.length > 0) profile.avatar = avatarPaths.map((avatarPath) => `https://images01-evilangel.gammacdn.com/actors${avatarPath}`);
@@ -929,7 +929,8 @@ async function fetchApiProfile({ name: actorName }, context, include) {
 	const siteSlug = context.entity.slug || context.site?.slug || context.network?.slug;
 
 	const actorSlug = encodeURI(actorName);
-	const referer = `https://www.${siteSlug}.com/en/search`;
+	// const referer = `https://www.${siteSlug}.com/en/search`;
+	const referer = `${new URL(context.entity.url).origin}/en/search`;
 
 	const { apiUrl } = await fetchApiCredentials(referer);
 

src/scrapers/scrapers.js

@@ -20,7 +20,7 @@ const czechav = require('./czechav');
 const modelmedia = require('./modelmedia');
 const dorcel = require('./dorcel');
 const fabulouscash = require('./fabulouscash');
-const famedigital = require('./famedigital');
+// const famedigital = require('./famedigital');
 const firstanalquest = require('./firstanalquest');
 const elevatedx = require('./elevatedx');
 const exploitedx = require('./exploitedx');
@@ -89,6 +89,10 @@ const freeones = require('./freeones');
 
 const scrapers = {
 	releases: {
+		// gamma
+		gamma,
+		// daringsex,
+		// etc
 		amateurallure,
 		americanpornstar,
 		amateureuro: porndoe,
@@ -114,13 +118,11 @@ const scrapers = {
 		delphine: modelmedia,
 		dorcel,
 		elegantangel: adultempire,
-		famedigital,
 		exploitedx,
 		fabulouscash,
 		firstanalquest,
 		forbondage: porndoe,
 		fullpornnetwork,
-		gamma,
 		gaywire: aylo,
 		girlgirl: julesjordan,
 		hitzefrei,
@@ -220,9 +222,40 @@ const scrapers = {
 		transangels: aylo,
 		trueamateurs: aylo,
 		twistys: aylo,
+		// gamma
+		'21sextury': gamma,
+		biphoria: gamma,
+		blakemason: gamma,
+		blowpass: gamma,
+		burningangel: gamma,
+		chaosmen: gamma,
+		devilsfilm: gamma,
+		diabolic: gamma,
+		dogfartnetwork: gamma,
+		evilangel: gamma,
+		famedigital: gamma,
+		fantasymassage: gamma,
+		filthykings: gamma,
+		gangbangcreampie: gamma,
+		girlsway: gamma,
+		gloryholesecrets: gamma,
+		peternorth: gamma,
+		pridestudios: gamma,
+		puretaboo: gamma,
+		roccosiffredi: gamma,
+		silverstonedvd: gamma,
+		silviasaint: gamma,
+		tabooheat: gamma,
+		vivid: gamma,
+		whiteghetto: gamma,
+		wicked: gamma,
+		xempire: gamma,
+		zerotolerancefilms: gamma,
+		'3rddegreefilms': gamma,
+		addicted2girls: gamma,
+		genderxfilms: gamma,
 		// etc
 		'18vr': badoink,
-		'21sextury': gamma,
 		adultempire,
 		allanal: mikeadriano,
 		allherluv: missax,
@@ -252,25 +285,19 @@ const scrapers = {
 		boobpedia,
 		bradmontana,
 		brattysis: nubiles,
-		burningangel: gamma,
 		cherrypimps,
 		cumlouder,
 		deeplush: nubiles,
-		devilsfilm: famedigital,
 		dirtyauditions: mikeadriano,
-		dogfartnetwork: gamma,
 		dorcelclub: dorcel,
 		doubleviewcasting: firstanalquest,
 		dtfsluts: fullpornnetwork,
-		evilangel: gamma,
 		exploitedx, // only from known URL that will specify site
 		eyeontheguy: hush,
 		firstanalquest,
 		forbondage: porndoe,
 		freeones,
-		gangbangcreampie: gamma,
 		girlfaction: fullpornnetwork,
-		gloryholesecrets: gamma,
 		gotfilled: radical,
 		hergape: fullpornnetwork,
 		hitzefrei,
@@ -316,7 +343,6 @@ const scrapers = {
 		pascalssubsluts,
 		pervcity,
 		pervertgallery: fullpornnetwork,
-		peternorth: famedigital,
 		pierrewoodman,
 		pimpxxx: cherrypimps,
 		porncz,
@@ -328,13 +354,10 @@ const scrapers = {
 		purgatoryx,
 		realvr: badoink,
 		rickysroom,
-		roccosiffredi: famedigital,
 		sayuncle: teamskeet,
 		score,
 		seehimfuck: hush,
 		sexlikereal,
-		silverstonedvd: famedigital,
-		silviasaint: famedigital,
 		spermmania: snowvalley,
 		handjobjapan: snowvalley,
 		fellatiojapan: snowvalley,
@@ -378,9 +401,7 @@ const scrapers = {
 		wifey: vixen,
 		vrcosplayx: badoink,
 		wankzvr,
-		wicked: gamma,
 		wildoncam: cherrypimps,
-		xempire: gamma,
 	},
 };
 

tests/profiles.js

@@ -15,9 +15,6 @@ const knex = require('../src/knex');
 const actors = [
 	// jules jordan
 	{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] },
-	// gamma
-	{ entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
-	{ entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
 	// vixen
 	{ entity: 'vixen', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
 	{ entity: 'tushy', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
@@ -81,6 +78,38 @@ const actors = [
 	{ entity: 'sweetsinner', name: 'Anna Claire Clouds', fields: ['avatar', 'gender'] },
 	// bangros
 	{ entity: 'bangbros', name: 'Kira Perez', fields: ['avatar', 'gender', 'ethnicity', 'hairColor'] },
+	// gamma
+	{ entity: '21sextury', name: 'Aletta Ocean', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
+	{ entity: 'biphoria', name: 'Cherry Kiss', fields: ['avatar', 'gender'] },
+	{ entity: 'blowpass', name: 'Jynx Maze', fields: ['avatar', 'description', 'gender'] },
+	{ entity: 'burningangel', name: 'Joanna Angel', fields: ['avatar', 'gender'] },
+	{ entity: 'chaosmen', name: 'Kenzo Alvarez', fields: ['avatar', 'gender'] },
+	{ entity: 'dogfartnetwork', name: 'Liz Jordan', fields: ['avatar', 'gender'] },
+	{ entity: 'diabolic', name: 'Kira Noir', fields: ['avatar', 'gender'] },
+	{ entity: 'evilangel', name: 'Francesca Le', fields: ['avatar', 'gender'] },
+	{ entity: 'fantasymassage', name: 'Cherry Kiss', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
+	{ entity: 'filthykings', name: 'Armani Black', fields: ['avatar', 'gender'] },
+	{ entity: 'gangbangcreampie', name: 'Luna Lovely', fields: ['avatar', 'gender', 'description'] },
+	{ entity: 'girlsway', name: 'Adriana Chechik', fields: ['avatar', 'gender', 'description', 'eyes', 'hairColor'] },
+	{ entity: 'gloryholesecrets', name: 'Maya Bijou', fields: ['avatar', 'gender'] },
+	{ entity: 'pridestudios', name: 'Peter Hooke', fields: ['avatar', 'gender'] },
+	{ entity: 'puretaboo', name: 'Jane Wilde', fields: ['avatar', 'gender'] },
+	{ entity: 'tabooheat', name: 'Cory Chase', fields: ['avatar', 'gender'] },
+	{ entity: 'vivid', name: 'Avi Love', fields: ['avatar', 'gender'] },
+	{ entity: 'wicked', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
+	{ entity: 'xempire', name: 'Abella Danger', fields: ['gender', 'avatar', 'description'] },
+	// gamma > zero tolerance / stod
+	{ entity: '3rddegreefilms', name: 'Angel Dark', fields: ['gender', 'avatar'] },
+	{ entity: 'addicted2girls', name: 'Abigail Mac', fields: ['gender', 'avatar'] },
+	{ entity: 'zerotolerancefilms', name: 'Courtney Cummz', fields: ['gender', 'avatar'] },
+	{ entity: 'genderxfilms', name: 'Kasey Kei', fields: ['gender', 'avatar'] },
+	// gamma > famedigital
+	{ entity: 'roccosiffredi', name: 'Malena Nazionale', fields: ['avatar', 'gender'] },
+	{ entity: 'famedigital', name: 'Hime Marie', fields: ['avatar', 'gender'] },
+	{ entity: 'peternorth', name: 'Carmen Caliente', fields: ['avatar', 'gender'] },
+	{ entity: 'silverstonedvd', name: 'Leanni Lei', fields: ['avatar', 'gender'] },
+	{ entity: 'silviasaint', name: 'Silvia Saint', fields: ['avatar', 'gender', 'description'] },
+	{ entity: 'whiteghetto', name: 'Proxy Paige', fields: ['avatar', 'gender', 'description'] },
 ];
 
 const actorScrapers = scrapers.actors;
@@ -183,7 +212,7 @@ async function init() {
 				}, include);
 
 				console.log(profile);
-				console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => field).join(', '));
+				console.log('Untested fields', Object.entries(profile).filter(([field, value]) => !actor.fields.includes(field) && typeof value !== 'undefined' && value !== null).map(([field]) => `'${field}'`).join(', '));
 
 				if (!profile) {
 					assert.fail('profile not found');