1.252.11

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.252.6",
+    "version": "1.252.11",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.252.6",
+            "version": "1.252.11",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",
@@ -95,7 +95,7 @@
                 "tunnel": "0.0.6",
                 "ua-parser-js": "^1.0.37",
                 "undici": "^7.24.7",
-                "unprint": "^0.19.13",
+                "unprint": "^0.19.20",
                 "url-pattern": "^1.0.3",
                 "v-tooltip": "^2.1.3",
                 "video.js": "^8.6.1",
@@ -20844,9 +20844,9 @@
             }
         },
         "node_modules/unprint": {
-            "version": "0.19.13",
-            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.13.tgz",
-            "integrity": "sha512-HPNCQn2CziiGeK0JSZg/5E+G2prHme+8lDojxd16wUwSQ0mgW4nZq4LOuVMIRRAFm1M1nkju0oMIdsj4uRFASw==",
+            "version": "0.19.20",
+            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.20.tgz",
+            "integrity": "sha512-Z1SVqsbs+CRS5VIRKKVmCYoki8q8gghY65lY+epxK4vMdT9rlouC9YGpHk0J9FusH0GQPIdG9MiWHIvAMGE3SA==",
             "dependencies": {
                 "bottleneck": "^2.19.5",
                 "cookie": "^1.1.1",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.252.6",
+    "version": "1.252.11",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {
@@ -154,7 +154,7 @@
         "tunnel": "0.0.6",
         "ua-parser-js": "^1.0.37",
         "undici": "^7.24.7",
-        "unprint": "^0.19.13",
+        "unprint": "^0.19.20",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.1.3",
         "video.js": "^8.6.1",

seeds/02_sites.js

@@ -7362,6 +7362,17 @@ const sites = [
 			studioId: 698,
 		},
 	},
+	{
+		name: 'Disciples of Desire',
+		slug: 'disciplesofdesire',
+		url: 'https://disciplesofdesire.com/',
+		parent: 'analvids',
+		independent: true,
+		parameters: {
+			usePornboxUrl: true,
+			studioId: 1214,
+		},
+	},
 	// LETS DOE IT
 	{
 		name: 'The White Boxxx',

seeds/06_affiliates.js

@@ -807,6 +807,7 @@ const affiliates = [
 		network: 'analvids',
 		parameters: {
 			query: 'aff=BW90MHT1DP', // supported
+			recursive: false,
 		},
 		comment: '50% rev share',
 	},

src/scrapers/aylo.js

@@ -1,15 +1,10 @@
 'use strict';
 
 /* eslint-disable newline-per-chained-call */
-const Promise = require('bluebird');
-const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
-const cookie = require('cookie');
 const moment = require('moment');
-// const unprint = require('unprint');
+const unprint = require('unprint');
 
-const qu = require('../utils/qu');
 const slugify = require('../utils/slugify');
-const http = require('../utils/http');
 const { inchesToCm, lbsToKg } = require('../utils/convert');
 
 function getBasePath(parameters, channel, path = '/scene') {
@@ -126,6 +121,119 @@ async function scrapeLatest(items, site, filterChannel, options) {
 	};
 }
 
+function getUrl(site) {
+	const { searchParams, pathname } = new URL(site.url);
+
+	// if (search.match(/\?site=\d+/)) {
+	if (searchParams.has('site') || /\/site\/\d+/.test(pathname)) {
+		return site.url;
+	}
+
+	if (site.parameters?.native) {
+		return `${site.url}/scenes`;
+	}
+
+	if (site.parameters?.extract) {
+		return `${site.url}/scenes`;
+	}
+
+	if (site.parameters?.siteId) {
+		return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
+	}
+
+	throw new Error(`Aylo site '${site.name}' (${site.url}) not supported`);
+}
+
+async function getSession(site, _parameters, url) {
+	// if (site.slug === 'aylo' || site.parameters?.parentSession === false) {
+	if (site.slug === 'aylo') {
+		// most MG sites have a parent network to acquire a session from, don't try to acquire session from mindgeek.com for independent channels
+		return null;
+	}
+
+	const sessionUrl = site.parameters?.siteId && !(site.parameters?.native || site.parameters?.childSession || site.parent?.parameters?.childSession || site.parameters?.parentSession === false)
+		? site.parent.url
+		: (url || site.url);
+
+	const res = await unprint.get(sessionUrl, {
+		headers: {
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+			Connection: 'keep-alive',
+		},
+	});
+
+	if (res.status === 200) {
+		const instanceToken = res.cookies.instance_token;
+
+		if (instanceToken) {
+			return { instanceToken };
+		}
+	}
+
+	throw new Error(`Failed to acquire Aylo session (${res.statusCode})`);
+}
+
+async function fetchLatest(site, page = 1, options) {
+	const url = getUrl(site);
+	const { searchParams, pathname } = new URL(url);
+	const siteId = searchParams.get('site') || Number(pathname.match(/\/site\/(\d+)\//)?.[1]);
+
+	if (!siteId && !site.parameters?.native && !site.parameters?.extract) {
+		return null;
+	}
+
+	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
+		? options.beforeNetwork
+		: await getSession(site, options.parameters, url);
+
+	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+	const limit = 24;
+	const apiUrl = site.parameters?.native || site.parameters?.extract
+		? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
+		: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
+
+	const res = await unprint.get(apiUrl, {
+		interval: options.parameters.interval,
+		concurrency: options.parameters.concurrency,
+		headers: {
+			Instance: instanceToken,
+			Origin: site.url,
+			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
+	});
+
+	if (res.status === 200 && res.data.result) {
+		return scrapeLatest(res.data.result, site, false, options);
+	}
+
+	return res.status;
+}
+
+async function fetchUpcoming(site, _page, options) {
+	const url = getUrl(site);
+	const { instanceToken } = await getSession(site, options.parameters);
+
+	const apiUrl = 'https://site-api.project1service.com/v2/upcoming-releases';
+
+	const res = await unprint.get(apiUrl, {
+		interval: options.parameters.interval,
+		concurrency: options.parameters.concurrency,
+		headers: {
+			Instance: instanceToken,
+			Origin: site.url,
+			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
+	});
+
+	if (res.status === 200 && res.data.result) {
+		return scrapeLatest(res.data.result, site, true, options);
+	}
+
+	return res.status;
+}
+
 function scrapeRelease(data, url, channel, networkName, options) {
 	if (Array.isArray(data)) {
 		return null;
@@ -192,139 +300,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	return release;
 }
 
-function getUrl(site) {
-	const { searchParams, pathname } = new URL(site.url);
-
-	// if (search.match(/\?site=\d+/)) {
-	if (searchParams.has('site') || /\/site\/\d+/.test(pathname)) {
-		return site.url;
-	}
-
-	if (site.parameters?.native) {
-		return `${site.url}/scenes`;
-	}
-
-	if (site.parameters?.extract) {
-		return `${site.url}/scenes`;
-	}
-
-	if (site.parameters?.siteId) {
-		return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
-	}
-
-	throw new Error(`Aylo site '${site.name}' (${site.url}) not supported`);
-}
-
-async function getSession(site, parameters, url) {
-	// if (site.slug === 'aylo' || site.parameters?.parentSession === false) {
-	if (site.slug === 'aylo') {
-		// most MG sites have a parent network to acquire a session from, don't try to acquire session from mindgeek.com for independent channels
-		return null;
-	}
-
-	const cookieJar = new CookieJar();
-	const session = http.session({ cookieJar });
-
-	const sessionUrl = site.parameters?.siteId && !(site.parameters?.native || site.parameters?.childSession || site.parent?.parameters?.childSession || site.parameters?.parentSession === false)
-		? site.parent.url
-		: (url || site.url);
-
-	/*
-	await unprint.browserRequest(sessionUrl, {
-		browser: {
-			headless: false,
-		},
-		async control() {
-			await new Promise((resolve) => { setTimeout(() => resolve(), 10000); });
-		},
-	});
-	*/
-
-	const res = await http.get(sessionUrl, {
-		session,
-		headers: {
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-			Connection: 'keep-alive',
-		},
-		interval: parameters?.interval,
-		concurrency: parameters?.concurrency,
-		parse: false,
-	});
-
-	if (res.status === 200) {
-		const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
-		const { instance_token: instanceToken } = cookie.parse(cookieString);
-
-		if (instanceToken) {
-			return { session, instanceToken };
-		}
-	}
-
-	throw new Error(`Failed to acquire Aylo session (${res.statusCode})`);
-}
-
-async function fetchLatest(site, page = 1, options) {
-	const url = getUrl(site);
-	const { searchParams, pathname } = new URL(url);
-	const siteId = searchParams.get('site') || Number(pathname.match(/\/site\/(\d+)\//)?.[1]);
-
-	if (!siteId && !site.parameters?.native && !site.parameters?.extract) {
-		return null;
-	}
-
-	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
-		? options.beforeNetwork
-		: await getSession(site, options.parameters, url);
-
-	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
-	const limit = 24;
-	const apiUrl = site.parameters?.native || site.parameters?.extract
-		? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
-		: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
-
-	const res = await http.get(apiUrl, {
-		interval: options.parameters.interval,
-		concurrency: options.parameters.concurrency,
-		headers: {
-			Instance: instanceToken,
-			Origin: site.url,
-			Referer: url,
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-		},
-	});
-
-	if (res.status === 200 && res.body.result) {
-		return scrapeLatest(res.body.result, site, false, options);
-	}
-
-	return res.statusCode;
-}
-
-async function fetchUpcoming(site, page, options) {
-	const url = getUrl(site);
-	const { session, instanceToken } = await getSession(site, options.parameters);
-
-	const apiUrl = 'https://site-api.project1service.com/v2/upcoming-releases';
-
-	const res = await http.get(apiUrl, {
-		session,
-		interval: options.parameters.interval,
-		concurrency: options.parameters.concurrency,
-		headers: {
-			Instance: instanceToken,
-			Origin: site.url,
-			Referer: url,
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-		},
-	});
-
-	if (res.statusCode === 200 && res.body.result) {
-		return scrapeLatest(res.body.result, site, true, options);
-	}
-
-	return res.statusCode;
-}
-
 async function fetchRelease(url, site, baseScene, options) {
 	if (baseScene?.entryId && !baseScene.shallow && !options.parameters.forceDeep) {
 		// overview and deep data is the same, don't hit server unnecessarily
@@ -332,10 +307,9 @@ async function fetchRelease(url, site, baseScene, options) {
 	}
 
 	const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
-	const { session, instanceToken } = options.beforeFetchScenes || await getSession(site, options.parameters);
+	const { instanceToken } = options.beforeFetchScenes || await getSession(site, options.parameters);
 
-	const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
-		session,
+	const res = await unprint.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
 		interval: options.parameters.interval,
 		concurrency: options.parameters.concurrency,
 		headers: {
@@ -344,16 +318,16 @@ async function fetchRelease(url, site, baseScene, options) {
 		},
 	});
 
-	if (res.status === 200 && res.body.result) {
+	if (res.status === 200 && res.data.result) {
 		return {
-			scene: scrapeRelease(res.body.result, url, site, null, options),
+			scene: scrapeRelease(res.data.result, url, site, null, options),
 		};
 	}
 
 	return null;
 }
 
-function scrapeProfile(data, networkName, _releases = []) {
+function scrapeProfile(data, _networkName, _releases = []) {
 	const profile = {
 		description: data.bio,
 		aliases: data.aliases.filter(Boolean),
@@ -367,7 +341,7 @@ function scrapeProfile(data, networkName, _releases = []) {
 		profile.measurements = data.measurements;
 	}
 
-	profile.dateOfBirth = qu.parseDate(data.birthday);
+	profile.dateOfBirth = unprint.extractDate(data.birthday);
 	profile.birthPlace = data.birthPlace;
 	profile.height = inchesToCm(data.height);
 	profile.weight = lbsToKg(data.weight);
@@ -406,10 +380,9 @@ function scrapeProfile(data, networkName, _releases = []) {
 
 async function fetchProfile({ name: actorName }, { entity, parameters }, include) {
 	// const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
-	const { session, instanceToken } = await getSession(entity, parameters);
+	const { instanceToken } = await getSession(entity, parameters);
 
-	const res = await http.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
-		session,
+	const res = await unprint.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
 		interval: parameters.interval,
 		concurrency: parameters.concurrency,
 		headers: {
@@ -418,14 +391,13 @@ async function fetchProfile({ name: actorName }, { entity, parameters }, include
 		},
 	});
 
-	if (res.statusCode === 200) {
-		const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
+	if (res.status === 200) {
+		const actorData = res.data.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
 
 		if (actorData) {
 			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
 
-			const actorReleasesRes = include.includeActorScenes && await http.get(actorReleasesUrl, {
-				session,
+			const actorReleasesRes = include.includeActorScenes && await unprint.get(actorReleasesUrl, {
 				interval: parameters.interval,
 				concurrency: parameters.concurrency,
 				headers: {
@@ -433,8 +405,8 @@ async function fetchProfile({ name: actorName }, { entity, parameters }, include
 				},
 			});
 
-			if (actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
-				return scrapeProfile(actorData, entity.slug, actorReleasesRes.body.result);
+			if (actorReleasesRes.status === 200 && actorReleasesRes.data.result) {
+				return scrapeProfile(actorData, entity.slug, actorReleasesRes.data.result);
 			}
 
 			return scrapeProfile(actorData, entity.slug, []);

src/scrapers/kink.js

@@ -18,7 +18,7 @@ function scrapeAll(scenes, entity) {
 		release.entryId = release.shootId;
 
 		release.title = query.content('.card-body a[href*="/shoot"]').trim();
-		release.date = query.date('small > span', 'MMM D, YYYY');
+		release.date = query.date('.small > span', 'MMM D, YYYY');
 
 		release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
@@ -54,7 +54,11 @@ function scrapeAll(scenes, entity) {
 			// no photos
 		}
 
-		release.trailer = `https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`;
+		release.trailer = [
+			query.video('.ratio-thumbnail img', { attribute: 'data-trailer-url' }),
+			`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
+			`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
+		];
 
 		release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
 		release.rating = query.number('.thumb-up') / 10;
@@ -64,10 +68,13 @@ function scrapeAll(scenes, entity) {
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `${channel.parent.url}/search?type=shoots&channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
+	const url = `${channel.parent.url}/shoots?channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
 
 	const res = await unprint.browser(url, {
 		selectAll: '.container .card',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -129,6 +136,7 @@ function scrapeScene({ query }, url, entity) {
 			src: source.url,
 			quality: source.resolution,
 		})) || []),
+		`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
 		`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
 	];
 
@@ -143,7 +151,11 @@ function scrapeScene({ query }, url, entity) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await unprint.browser(url);
+	const res = await unprint.browser(url, {
+		cookies: {
+			age_gate_accepted: '1',
+		},
+	});
 
 	if (res.status === 200) {
 		const scene = scrapeScene(res.context, url, channel);
@@ -197,6 +209,9 @@ async function getActorUrl({ name: actorName, url }, networkUrl) {
 	// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
 	const res = await unprint.get(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`, {
 		interface: 'request',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -217,7 +232,11 @@ async function fetchProfile(actor, entity) {
 	const actorUrl = await getActorUrl(actor, networkUrl);
 
 	if (actorUrl) {
-		const actorRes = await unprint.browser(actorUrl);
+		const actorRes = await unprint.browser(actorUrl, {
+			cookies: {
+				age_gate_accepted: '1',
+			},
+		});
 
 		if (actorRes.status === 200) {
 			return scrapeProfile(actorRes.context, actorUrl);

src/scrapers/nubiles.js

@@ -21,9 +21,9 @@ function stripQuery(link) {
 }
 
 async function getPhotos(albumUrl) {
-	const res = await unprint.get(albumUrl, {
+	// aggressive Turnstile protection calls for browser
+	const res = await unprint.browser(albumUrl, {
 		selectAll: '.photo-thumb',
-		interface: 'request',
 	});
 
 	return res.ok

src/scrapers/pornbox.js

@@ -33,7 +33,7 @@ function extractShootId(title) {
 	return title.trim().match(/[A-Z]{2,3}\d{3,4}\w?/)?.[0].toUpperCase();
 }
 
-async function scrapeScene(data, channel, include) {
+async function scrapeScene(data, channel, context) {
 	const release = {};
 	const entityUrl = new URL(channel.url).origin;
 
@@ -41,7 +41,10 @@ async function scrapeScene(data, channel, include) {
 
 	release.title = data.scene_name || data.custom_name;
 
-	release.url = `${entityUrl}/watch/${data.id}/${slugify(release.title, '_') || ''}`;
+	release.url = context.parameters?.usePornboxUrl
+		? `https://pornbox.com/application/watch-page/${data.id}`
+		: `${entityUrl}/watch/${data.id}/${slugify(release.title, '_') || ''}`;
+
 	release.shootId = extractShootId(release.title);
 
 	release.date = new Date(data.release_date || data.publish_date);
@@ -77,7 +80,7 @@ async function scrapeScene(data, channel, include) {
 
 	release.teaser = data.video_preview;
 
-	if (include?.includeTrailers) {
+	if (context?.includeTrailers) {
 		release.trailer = await getTrailer(data);
 		release.qualities = release.trailer?.map((src) => src.quality);
 	}
@@ -91,11 +94,11 @@ async function scrapeScene(data, channel, include) {
 	return release;
 }
 
-function scrapeAll(scenes, channel) {
-	return Promise.all(scenes.map(async (data) => scrapeScene(data, channel)));
+function scrapeAll(scenes, channel, context) {
+	return Promise.all(scenes.map(async (data) => scrapeScene(data, channel, context)));
 }
 
-async function fetchLatest(channel, page = 1) {
+async function fetchLatest(channel, page = 1, context) {
 	if (!channel.parameters?.studioId) {
 		return null;
 	}
@@ -104,18 +107,18 @@ async function fetchLatest(channel, page = 1) {
 	const res = await unprint.get(url);
 
 	if (res.ok && res.data?.contents) {
-		return scrapeAll(res.data.contents, channel);
+		return scrapeAll(res.data.contents, channel, context);
 	}
 
 	return res.status;
 }
 
-async function fetchScene(url, entity, baseRelease, include) {
+async function fetchScene(url, entity, baseRelease, context) {
 	const entryId = new URL(url).pathname.match(/\/watch(?:-page)?\/(\d+)/)[1];
 	const res = await unprint.get(`https://pornbox.com/contents/${entryId}`);
 
 	if (res.ok) {
-		return scrapeScene(res.data, entity, include);
+		return scrapeScene(res.data, entity, context);
 	}
 
 	return res.status;

src/scrapers/releases.js

@@ -97,6 +97,7 @@ module.exports = {
 	familystrokes: teamskeet,
 	// etc
 	analvids: pornbox,
+	pornbox,
 	kellymadison,
 	kink,
 	'8kmembers': kellymadison,

tests/profiles.js

@@ -213,7 +213,7 @@ const actors = [
 	{ entity: 'naughtyamerica', name: 'Nicole Aniston', fields: ['avatar', 'description'] },
 	{ entity: 'tonightsgirlfriend', name: 'Abella Danger', fields: ['avatar'] },
 	// jules jordan scraper
-	{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] },
+	{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'avatar'] },
 	{ entity: 'amateurallure', name: 'Ava Amira', fields: ['avatar', 'description'] },
 	{ entity: 'swallowsalon', name: 'Abella Danger', fields: ['avatar'] },
 	// exploitedx