1.248.33

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.248.32",
+    "version": "1.248.33",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.248.32",
+            "version": "1.248.33",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.248.32",
+    "version": "1.248.33",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/scrapers/actors.js

@@ -2,7 +2,7 @@
 
 const adultempire = require('./adultempire');
 const angelogodshackoriginal = require('./angelogodshackoriginal');
-const americanpornstar = require('./americanpornstar');
+// const americanpornstar = require('./americanpornstar'); // offline
 const aziani = require('./aziani');
 const badoink = require('./badoink');
 const bamvisions = require('./bamvisions');
@@ -163,10 +163,10 @@ module.exports = {
 	thatsitcomshow: nubiles,
 	// porndoe
 	amateureuro: porndoe,
-	forbondage: porndoe,
 	mamacitaz: porndoe,
 	transbella: porndoe,
 	vipsexvault: porndoe,
+	// forbondage: porndoe,
 	// aziani
 	aziani,
 	'2poles1hole': aziani,
@@ -209,7 +209,7 @@ module.exports = {
 	fullpornnetwork,
 	adultempire,
 	allherluv: missax,
-	americanpornstar,
+	// americanpornstar,
 	angelogodshackoriginal,
 	babevr: badoink,
 	badoinkvr: badoink,

src/scrapers/adultempire.js

@@ -2,9 +2,8 @@
 
 const unprint = require('unprint');
 
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
-const { feetInchesToCm, lbsToKg } = require('../utils/convert');
+const { convert } = require('../utils/convert');
 
 function scrapeAll(scenes, channel, _options) {
 	return scenes.map(({ query }) => {
@@ -122,18 +121,18 @@ async function scrapeProfile({ query }) {
 	const bioText = query.content('#profileModal .well');
 
 	profile.description = query.content('#profileModal .modal-body')
-		.slice(bioText.length)
+		?.slice(bioText?.length || 0)
 		.replace(/Biography Text ©Adult DVD Empire/i, '')
 		.trim();
 
 	profile.measurements = bio.measurements?.replace(/["\s]+/g, '');
 
 	profile.hair = bio.hair;
-	profile.eyes = bio.eyes;
+	profile.eyes = bio.eyes?.replace(/eyes?/i, '').trim();
 	profile.ethnicity = bio.ethnicity;
 
-	profile.height = feetInchesToCm(bio.height);
-	profile.weight = lbsToKg(bio.weight);
+	profile.height = convert(bio.height, 'cm');
+	profile.weight = convert(bio.weight, 'lb', 'kg');
 
 	const avatar = query.img('picture img, .performer-image-container img');
 
@@ -155,8 +154,8 @@ async function fetchLatest(channel, page, options) {
 		? `${options.parameters.latest}?page=${page}&view=grid`
 		: `${channel.url}/watch-newest-clips-and-scenes.html?page=${page}&view=grid`, {
 		selectAll: '.item-grid-scene .grid-item',
-		headers: {
-			Cookie: 'ageConfirmed=true;',
+		cookies: {
+			ageConfirmed: true,
 		},
 	});
 
@@ -171,6 +170,9 @@ async function fetchProfilePage(actorUrl) {
 	const res = await unprint.get(actorUrl, {
 		select: '#content',
 		rejectUnauthorized: false,
+		cookies: {
+			ageConfirmed: true,
+		},
 	});
 
 	if (res.ok) {
@@ -189,10 +191,14 @@ async function fetchProfile(baseActor, channel, include) {
 		}
 	}
 
-	const searchRes = await http.get(`https://www.adultempire.com/search/SearchAutoComplete_Agg_EmpireDTRank?search_type=Pornstars&rows=9&name_startsWith=${slugify(baseActor.name, '+')}`);
+	const searchRes = await unprint.get(`https://www.adultempire.com/search/SearchAutoComplete_Agg_EmpireDTRank?search_type=Pornstars&rows=9&name_startsWith=${slugify(baseActor.name, '+')}`, {
+		cookies: {
+			ageConfirmed: true,
+		},
+	});
 
-	if (searchRes.ok && searchRes.body.Results) {
-		const actorResult = searchRes.body.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
+	if (searchRes.ok && searchRes.data.Results) {
+		const actorResult = searchRes.data.Results.find((result) => /performer/i.test(result.BasicResponseGroup?.displaytype) && new RegExp(baseActor.name, 'i').test(result.BasicResponseGroup?.description));
 
 		if (actorResult) {
 			const url = `https://www.adultempire.com/${actorResult.BasicResponseGroup.id}`;

src/scrapers/freeones.js

@@ -1,91 +1,115 @@
 'use strict';
 
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
+const unprint = require('unprint');
 
-const http = require('../utils/http');
+const slugify = require('../utils/slugify');
 
-function scrapeProfile(html, actorName) {
-	const { document } = new JSDOM(html).window;
-	const profile = { name: actorName };
+function scrapeProfile({ query }) {
+	const profile = {};
 
-	const bio = Array.from(document.querySelectorAll('a[href^="/babes"]'), (el) => decodeURI(el.href)).reduce((acc, item) => {
-		const keyMatch = item.match(/\[\w+\]/);
+	const bio = Object.fromEntries(query.all('.profile-meta-list li').map((bioEl) => [
+		slugify(unprint.query.content(bioEl, 'span:first-child'), '_'),
+		unprint.query.content(bioEl, 'span:last-child'),
+	]).filter(([_key, value]) => value?.toLowerCase() !== 'unknown'));
 
-		if (keyMatch) {
-			const key = keyMatch[0].slice(1, -1);
-			const [, value] = item.split('=');
+	profile.description = query.content('#description div[data-test="biography"]');
 
-			// both hip and waist link to 'waist', assume biggest value is hip
-			if (key === 'waist' && acc.waist) {
-				if (acc.waist > value) {
-					acc.hip = acc.waist;
-					acc.waist = value;
+	profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
+	profile.age = unprint.extractNumber(bio.age);
 
-					return acc;
-				}
-
-				acc.hip = value;
-
-				return acc;
-			}
-
-			acc[key] = value;
-		}
-
-		return acc;
-	}, {});
-
-	if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
-
-	if (bio.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
-	else if (bio.country) profile.birthPlace = bio.country;
-
-	profile.eyes = bio.eyeColor;
-	profile.hair = bio.hairColor;
+	profile.birthPlace = bio.place_of_birth;
+	profile.nationality = bio.nationality;
 	profile.ethnicity = bio.ethnicity;
 
-	profile.bust = bio.bra;
-	if (bio.waist) profile.waist = Number(bio.waist.split(',')[0]);
-	if (bio.hip) profile.hip = Number(bio.hip.split(',')[0]);
+	profile.eyes = bio.eye_color;
+	profile.hairColor = bio.hair_color;
 
-	if (bio.height) profile.height = Number(bio.height.split(',')[0]);
-	if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]);
+	[profile.bust, profile.cup] = bio.bra?.match(/(\d+)([a-z]+)/i)?.slice(1) || [];
 
-	profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), (el) => el.href);
+	// TODO: differentiate between bust and bra band size
+	if (!profile.bust) {
+		profile.bust = bio.bust;
+	}
 
-	const avatar = document.querySelector('.profile-image-large img').src;
-	if (!avatar.match('placeholder')) profile.avatar = { src: avatar, credit: null };
+	if (!profile.cup) {
+		profile.cup = bio.cup;
+	}
+
+	profile.bust = unprint.extractNumber(bio.bra);
+	profile.cup = bio.cup;
+	profile.waist = unprint.extractNumber(bio.waist);
+	profile.hip = unprint.extractNumber(bio.hip);
+
+	profile.height = unprint.extractNumber(bio.height);
+	profile.weight = unprint.extractNumber(bio.weight);
+
+	profile.foot = unprint.extractNumber(bio.shoe_size);
+
+	profile.socials = query.urls('.profile-meta-item .teaser__link');
+
+	if (/yes/i.test(bio.tattoos)) profile.hasTattoos = true;
+	if (/no/i.test(bio.tattoos)) profile.hasTattoos = false;
+
+	profile.tattoos = bio.tattoo_locations;
+
+	if (/yes/i.test(bio.piercings)) profile.hasPiercings = true;
+	if (/no/i.test(bio.piercings)) profile.hasPiercings = false;
+
+	profile.piercings = bio.piercing_locations;
+
+	if (/natural/i.test(bio.boobs)) profile.naturalBoobs = true;
+	if (/fake/i.test(bio.boobs)) profile.naturalBoobs = false;
+
+	if (/natural/i.test(bio.butt)) profile.naturalButt = true;
+	if (/fake/i.test(bio.butt)) profile.naturalButt = false;
+
+	const avatar = query.img('.dashboard-image-container img');
+
+	if (!avatar?.match(/placeholder/i)) {
+		profile.avatar = avatar;
+	}
 
 	return profile;
 }
 
-function scrapeSearch(html) {
-	const { document } = new JSDOM(html).window;
-
-	return document.querySelector('a.image-link')?.href || null;
-}
-
-async function fetchProfile({ name: actorName }) {
-	const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
-
-	const res = await http.get(`https://freeones.nl/${actorSlug}/profile`);
-
-	if (res.statusCode === 200) {
-		return scrapeProfile(res.body.toString(), actorName);
+async function getActorUrl(actor) {
+	if (actor.url) {
+		return actor.url;
 	}
 
-	const searchRes = await http.get(`https://freeones.nl/babes?q=${actorName}`);
-	const actorPath = scrapeSearch(searchRes.body.toString());
+	const res = await unprint.post('https://www.freeones.com/xhr/search', {
+		performerTypes: ['babe', 'male', 'trans'],
+		query: actor.name,
+		recipe: 'subject',
+		size: 12,
+	});
 
-	if (actorPath) {
-		const actorRes = await http.get(`https://freeones.nl${actorPath}/profile`);
+	if (res.ok) {
+		const model = res.data.hits?.find((result) => slugify(result.name) === actor.slug);
 
-		if (actorRes.statusCode === 200) {
-			return scrapeProfile(actorRes.body.toString(), actorName);
+		if (model?.url) {
+			return `https://www.freeones.com${model.url}/bio`;
 		}
+	}
 
-		return null;
+	return null;
+}
+
+async function fetchProfile(actor) {
+	const res = await unprint.get(`https://freeones.com/${actor.slug}/bio`);
+
+	if (res.ok) {
+		return scrapeProfile(res.context);
+	}
+
+	const actorUrl = await getActorUrl(actor);
+
+	if (actorUrl) {
+		const actorRes = await unprint.get(actorUrl);
+
+		if (actorRes.ok) {
+			return scrapeProfile(actorRes.context);
+		}
 	}
 
 	return null;

src/scrapers/modelmedia.js

@@ -30,7 +30,7 @@ function scrapeSceneApi(scene, channel, parameters) {
 			model.avatar,
 			model.avatar?.replace('_compressed', ''), // this is often a wider image, not just uncompressed
 		])).filter(Boolean),
-	}));
+	})).filter((actor) => actor.name?.toLowerCase() === 'amateur'); // generic name for various amateur models
 
 	release.tags = scene.tags?.map((tag) => tag.name);
 
@@ -131,6 +131,10 @@ function scrapeAll(scenes) {
 function scrapeProfileApi(model, channel, parameters) {
 	const profile = {};
 
+	if (model.name?.toLowerCase() === 'amateur') {
+		return null; // generic profile for various amateur models
+	}
+
 	profile.entryId = model.id;
 	profile.url = `${channel.origin}${parameters.basePath || ''}/models/${model.id}`;
 

tests/profiles.js

@@ -242,11 +242,15 @@ const actors = [
 	{ entity: 'boyfun', name: 'Amahd Passer', fields: ['avatar', 'age', 'height', 'weight', 'penisLength', 'isCircumcised'] },
 	{ entity: 'bang', name: 'Riley Reid', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'ethnicity', 'hairColor', 'eyes'] },
 	{ entity: 'littlecapricedreams', name: 'Littlecaprice', fields: ['avatar', 'nationality', 'cup', 'measurements', 'height', 'description'] }, // sic
-	{ entity: 'pascalssubsluts', name: 'Zlata Shine', fields: ['avatar', 'gender', 'nationality', 'hairColor', 'height', 'description'] }, // sic
-	{ entity: 'nebraskacoeds', name: 'Mary Beth Haglin', fields: ['avatar'] }, // sic
-	{ entity: 'firstanalquest', name: 'Abigaile Johnson', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'weight', 'height', 'measurements'] }, // sic
-	{ entity: 'doubleviewcasting', name: 'Abigaile Johnson', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'weight', 'height', 'measurements'] }, // sic
-	{ entity: 'boobpedia', name: 'Paige British', fields: ['avatar'] }, // sic
+	{ entity: 'pascalssubsluts', name: 'Zlata Shine', fields: ['avatar', 'gender', 'nationality', 'hairColor', 'height', 'description'] },
+	{ entity: 'nebraskacoeds', name: 'Mary Beth Haglin', fields: ['avatar'] },
+	{ entity: 'firstanalquest', name: 'Abigaile Johnson', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'weight', 'height', 'measurements'] },
+	{ entity: 'doubleviewcasting', name: 'Abigaile Johnson', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'weight', 'height', 'measurements'] },
+	{ entity: 'boobpedia', name: 'Paige British', fields: ['avatar'] },
+	{ entity: 'angelogodshackoriginal', name: 'Emily Pink', fields: ['avatar'] },
+	{ entity: 'bradmontana', name: 'Alicia Ribeiro', fields: ['avatar', 'gender'] },
+	{ entity: 'adultempire', name: 'Abella Danger', fields: ['avatar', 'description', 'measurements', 'eyes', 'height', 'weight'] },
+	{ entity: 'freeones', name: 'Sophia Locke', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'birthPlace', 'nationality', 'ethnicity', 'eyes', 'hairColor', 'bust', 'cup', 'waist', 'hip', 'height', 'weight', 'foot', 'socials', 'hasTattoos', 'tattoos', 'hasPiercings', 'piercings', 'naturalBoobs'] },
 ];
 
 const actorScrapers = scrapers.actors;
@@ -288,7 +292,7 @@ const validators = {
 	height: (value) => !!Number(value) && value > 130,
 	weight: (value) => !!Number(value) && value > 40,
 	eyes: (value) => typeof value === 'string' && value.length > 3,
-	hairColor: (value) => typeof value === 'string' && value.length > 3,
+	hairColor: (value) => typeof value === 'string' && value.length > 2,
 	measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module
 	dateOfBirth: (value) => value instanceof Date && !Number.isNaN(value.getFullYear()),
 	hasTattoos: (value) => typeof value === 'boolean',