1.214.0

assets/components/releases/release.vue

@@ -203,6 +203,19 @@
 				</div>
 			</div>
 
+			<div
+				v-if="release.qualities"
+				class="row"
+			>
+				<span class="row-label">Available qualities</span>
+
+				<span
+					v-for="quality in release.qualities"
+					:key="quality"
+					class="quality"
+				>{{ quality }}</span>
+			</div>
+
 			<div
 				v-if="release.comment"
 				class="row"
@@ -470,6 +483,16 @@ export default {
 	text-overflow: ellipsis;
 }
 
+.quality {
+	&::after {
+		content: 'p, ';
+	}
+
+	&:last-child::after {
+		content: 'p',
+	}
+}
+
 .releases {
 	margin: 0 0 .5rem 0;
 }

assets/js/config/default.js

@@ -12,6 +12,7 @@ export default {
 	selectableTags: [
 		'airtight',
 		'anal',
+		'bdsm',
 		'blowbang',
 		'blowjob',
 		'creampie',

assets/js/fragments.js

@@ -367,6 +367,7 @@ const releaseFields = `
     date
 	datePrecision
     slug
+	qualities
 	shootId
 	productionDate
 	comment
@@ -475,6 +476,7 @@ const releaseFragment = `
     duration
     createdAt
     shootId
+	qualities
 	productionDate
 	createdBatchId
 	productionLocation

config/default.js

@@ -89,6 +89,10 @@ module.exports = {
 			'uksinners',
 			// mindgeek
 			'pornhub',
+			// insex
+			'paintoy',
+			'aganmedon',
+			'sensualpain',
 		],
 		networks: [
 			// dummy network for testing

migrations/20220331135618_qualities.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.specificType('qualities', 'text[]');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.dropColumn('qualities');
+});

migrations/20220403235645_last_login.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.datetime('last_login');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.dropColumn('last_login');
+});

migrations/_20220330230122_stats.js

@@ -0,0 +1,25 @@
+exports.up = async (knex) => knex.raw(`
+	CREATE MATERIALIZED VIEW entities_stats
+	AS
+		WITH RECURSIVE relations AS (
+			SELECT entities.id, entities.parent_id, count(releases.id) AS releases_count, count(releases.id) AS total_count
+			FROM entities
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+
+			UNION ALL
+
+			SELECT entities.id AS entity_id, count(releases.id) AS releases_count, count(releases.id) + relations.total_count AS total_count
+			FROM entities
+			INNER JOIN relations ON relations.id = entities.parent_id
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+		)
+
+		SELECT relations.id AS entity_id, relations.releases_count
+		FROM relations;
+`);
+
+exports.down = async (knex) => knex.raw(`
+	DROP MATERIALIZED VIEW entities_stats;
+`);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.214.0",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.212.9",
+            "version": "1.214.0",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",
@@ -11650,25 +11650,6 @@
                 "webidl-conversions": "^3.0.0"
             }
         },
-        "node_modules/node-fetch/node_modules/tr46": {
-            "version": "0.0.3",
-            "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-            "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
-        },
-        "node_modules/node-fetch/node_modules/webidl-conversions": {
-            "version": "3.0.1",
-            "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-            "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
-        },
-        "node_modules/node-fetch/node_modules/whatwg-url": {
-            "version": "5.0.0",
-            "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-            "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
-            "dependencies": {
-                "tr46": "~0.0.3",
-                "webidl-conversions": "^3.0.0"
-            }
-        },
         "node_modules/node-gyp": {
             "version": "7.1.2",
             "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.214.0",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/02_sites.js

@@ -2714,163 +2714,142 @@ const sites = [
 	{
 		slug: 'blacksonblondes',
 		name: 'Blacks On Blondes',
-		url: 'https://www.blacksonblondes.com/tour',
+		url: 'https://www.blacksonblondes.com',
 		description: 'Blacks On Blondes is the Worlds Largest and Best Interracial Sex and Interracial Porn website. Black Men and White Women. BlacksOnBlondes has 23 years worth of Hardcore Interracial Content.  Featuring the entire Legendary Dogfart Movie Archive',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cuckoldsessions',
 		name: 'Cuckold Sessions',
-		url: 'https://www.cuckoldsessions.com/tour',
+		url: 'https://www.cuckoldsessions.com',
-		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com/tour - Hardcore Cuckold Fetish Videos',
+		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com - Hardcore Cuckold Fetish Videos',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryhole',
 		name: 'Glory Hole',
-		url: 'https://www.gloryhole.com/tour',
+		url: 'https://www.gloryhole.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksoncougars',
 		name: 'Blacks On Cougars',
-		url: 'https://www.blacksoncougars.com/tour',
+		url: 'https://www.blacksoncougars.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wefuckblackgirls',
 		name: 'We Fuck Black Girls',
 		alias: ['wfbg'],
-		url: 'https://www.wefuckblackgirls.com/tour',
+		url: 'https://www.wefuckblackgirls.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'watchingmymomgoblack',
 		name: 'Watching My Mom Go Black',
-		url: 'https://www.watchingmymomgoblack.com/tour',
+		url: 'https://www.watchingmymomgoblack.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialblowbang',
 		name: 'Interracial Blowbang',
-		url: 'https://www.interracialblowbang.com/tour',
+		url: 'https://www.interracialblowbang.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cumbang',
 		name: 'Cumbang',
-		url: 'https://www.cumbang.com/tour',
+		url: 'https://www.cumbang.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialpickups',
 		name: 'Interracial Pickups',
-		url: 'https://www.interracialpickups.com/tour',
+		url: 'https://www.interracialpickups.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'watchingmydaughtergoblack',
 		name: 'Watching My Daughter Go Black',
-		url: 'https://www.watchingmydaughtergoblack.com/tour',
+		url: 'https://www.watchingmydaughtergoblack.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'zebragirls',
 		name: 'Zebra Girls',
-		url: 'https://www.zebragirls.com/tour',
+		url: 'https://www.zebragirls.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryholeinitiations',
 		name: 'Gloryhole Initiations',
-		url: 'https://www.gloryhole-initiations.com/tour',
+		url: 'https://www.gloryhole-initiations.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'dogfartbehindthescenes',
 		name: 'Dogfart Behind The Scenes',
-		url: 'https://www.dogfartbehindthescenes.com/tour',
+		url: 'https://www.dogfartbehindthescenes.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blackmeatwhitefeet',
 		name: 'Black Meat White Feet',
-		url: 'https://www.blackmeatwhitefeet.com/tour',
+		url: 'https://www.blackmeatwhitefeet.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'springthomas',
 		name: 'Spring Thomas',
-		url: 'https://www.springthomas.com/tour',
+		url: 'https://www.springthomas.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'katiethomas',
 		name: 'Katie Thomas',
-		url: 'https://www.katiethomas.com/tour',
+		url: 'https://www.katiethomas.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'ruthblackwell',
 		name: 'Ruth Blackwell',
-		url: 'https://www.ruthblackwell.com/tour',
+		url: 'https://www.ruthblackwell.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'candymonroe',
 		name: 'Candy Monroe',
-		url: 'https://www.candymonroe.com/tour',
+		url: 'https://www.candymonroe.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wifewriting',
 		name: 'Wife Writing',
-		url: 'https://www.wifewriting.com/tour',
+		url: 'https://www.wifewriting.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'barbcummings',
 		name: 'Barb Cummings',
-		url: 'https://www.barbcummings.com/tour',
+		url: 'https://www.barbcummings.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'theminion',
 		name: 'The Minion',
-		url: 'https://www.theminion.com/tour',
+		url: 'https://www.theminion.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksonboys',
 		name: 'Blacks On Boys',
-		url: 'https://www.blacksonboys.com/tour',
+		url: 'https://www.blacksonboys.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryholesandhandjobs',
 		name: 'Gloryholes And Handjobs',
-		url: 'https://www.gloryholesandhandjobs.com/tour',
+		url: 'https://www.gloryholesandhandjobs.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	// DORCEL
@@ -4219,7 +4198,6 @@ const sites = [
 		tags: ['bdsm'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.sexuallybroken.com/sb',
 		},
 	},
@@ -4230,13 +4208,20 @@ const sites = [
 		url: 'https://www.infernalrestraints.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.infernalrestraints.com/ir',
+		},
 	},
 	{
 		slug: 'hardtied',
 		name: 'Hardtied',
+		alias: ['ht'],
 		url: 'https://www.hardtied.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.hardtied.com/ht',
+		},
 	},
 	{
 		slug: 'realtimebondage',
@@ -4245,6 +4230,9 @@ const sites = [
 		url: 'https://www.realtimebondage.com',
 		tags: ['bdsm', 'live'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.realtimebondage.com/rtb',
+		},
 	},
 	{
 		slug: 'topgrl',
@@ -4254,7 +4242,6 @@ const sites = [
 		tags: ['bdsm', 'femdom'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.topgrl.com/tg',
 		},
 	},
@@ -6909,6 +6896,13 @@ const sites = [
 			tourId: 9,
 		},
 	},
+	{
+		slug: 'dpdiva',
+		name: 'DP Diva',
+		url: 'http://dpdiva.com',
+		parent: 'pervcity',
+		tags: ['dp', 'anal'],
+	},
 	// PIERRE WOODMAN
 	{
 		slug: 'woodmancastingx',

src/app.js

@@ -85,23 +85,6 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
-async function startMemorySample() {
-	await inspector.heap.enable();
-	await inspector.heap.startSampling();
-
-	// monitorMemory();
-
-	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
-
-	setTimeout(async () => {
-		await stopMemorySample();
-
-		if (!done) {
-			await startMemorySample();
-		}
-	}, 30000);
-}
-
 async function init() {
 	try {
 		if (argv.server) {

src/argv.js

@@ -194,6 +194,7 @@ const { argv } = yargs
 		alias: 'pics',
 	})
 	.option('videos', {
+		alias: 'video',
 		describe: 'Include any trailers or teasers',
 		type: 'boolean',
 		default: true,

src/auth.js

@@ -34,6 +34,10 @@ async function login(credentials) {
 
 	await verifyPassword(credentials.password, user.password);
 
+	await knex('users')
+		.update('last_login', 'NOW()')
+		.where('id', user.id);
+
 	return curateUser(user);
 }
 

src/media.js

@@ -21,6 +21,7 @@ const argv = require('./argv');
 const knex = require('./knex');
 const http = require('./utils/http');
 const bulkInsert = require('./utils/bulk-insert');
+const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 
 const pipeline = util.promisify(stream.pipeline);
@@ -63,10 +64,10 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 		? chunks.slice(0, -1).concat(chunks.slice(-1).reverse())
 		: chunks;
 
-	const groupedMedias = lastPreferredChunks.map((chunk) => {
+	const groupedMedias = lastPreferredChunks.map((mediaChunk) => {
 		// merge chunked medias into single media with grouped fallback priorities,
 		// so the first sources of each media is preferred over all second sources, etc.
-		const sources = chunk
+		const sources = mediaChunk
 			.reduce((accSources, media) => {
 				media.sources.forEach((source, index) => {
 					if (!accSources[index]) {
@@ -82,8 +83,8 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 			.flat();
 
 		return {
-			id: chunk[0].id,
+			id: mediaChunk[0].id,
-			role: chunk[0].role,
+			role: mediaChunk[0].role,
 			sources,
 		};
 	});
@@ -235,22 +236,41 @@ async function findSourceDuplicates(baseMedias) {
 		.filter(Boolean);
 
 	const [existingSourceMedia, existingExtractMedia] = await Promise.all([
-		knex('media').whereIn('source', sourceUrls),
+		// my try to check thousands of URLs at once, don't pass all of them to a single query
-		knex('media').whereIn('source_page', extractUrls),
+		chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
+		chunk(extractUrls).reduce(async (chain, extractUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source_page', extractUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
 	]);
 
 	const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
 	const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
-	return { existingSourceMediaByUrl, existingExtractMediaByUrl };
+	return {
+		existingSourceMediaByUrl,
+		existingExtractMediaByUrl,
+	};
 }
 
 async function findHashDuplicates(medias) {
 	const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
 
-	const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
+	const existingHashMediaEntries = await chunk(hashes, 2).reduce(async (chain, hashesChunk) => {
-	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
+		const accHashes = await chain;
+		const existingHashes = await knex('media').whereIn('hash', hashesChunk);
 
+		return [...accHashes, ...existingHashes];
+	}, []);
+
+	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
 	const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
 
 	const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
@@ -600,11 +620,11 @@ async function fetchSource(source, baseMedia) {
 			const hashStream = new stream.PassThrough();
 			let size = 0;
 
-			hashStream.on('data', (chunk) => {
+			hashStream.on('data', (streamChunk) => {
-				size += chunk.length;
+				size += streamChunk.length;
 
 				if (hasherReady) {
-					hasher.write(chunk);
+					hasher.write(streamChunk);
 				}
 			});
 

src/scrapers/dogfart.js

@@ -1,20 +1,16 @@
 'use strict';
 
-/* eslint-disable newline-per-chained-call */
-// const Promise = require('bluebird');
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const qu = require('../utils/qu');
 
 async function getPhotos(albumUrl) {
-	const res = await http.get(albumUrl);
+	const res = await qu.get(albumUrl);
-	const html = res.body.toString();
-	const { document } = new JSDOM(html).window;
 
-	const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
+	if (!res.ok) {
+		return [];
+	}
+
+	const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1);
 	const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
 
 	const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
@@ -29,125 +25,95 @@ async function getPhotos(albumUrl) {
 	return photoUrls;
 }
 
-function scrapeLatest(html, site, filter = true) {
+function scrapeLatest(scenes, site, filter = true) {
-	const { document } = new JSDOM(html).window;
+	return scenes.reduce((acc, { query }) => {
-	const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
+		const release = {};
 
-	return sceneElements.map((element) => {
+		const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name');
-		const siteUrl = element.querySelector('.recent-details-title .help-block, .model-details-title .site-name').textContent;
 
-		if (filter && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
+		release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url });
+		release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`;
+
+		release.title = query.cnt('.scene-title');
+		// release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com
+
+		// release.poster = `https:${element.querySelector('img').src}`;
+		release.poster = query.img();
+		release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url');
+
+		release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
+
+		if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
 			// different dogfart site
-			return null;
+			return { ...acc, unextracted: [...acc.unextracted, release] };
 		}
 
-		const sceneLinkElement = element.querySelector('.thumbnail');
+		return { ...acc, scenes: [...acc.scenes, release] };
-		const url = qu.prefixUrl(sceneLinkElement.href, 'https://dogfartnetwork.com');
+	}, {
-		const { pathname } = new URL(url);
+		scenes: [],
-		const entryId = `${site.slug}_${pathname.split('/')[4]}`;
+		unextracted: [],
-
+	});
-		const title = element.querySelector('.scene-title').textContent;
-		const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
-
-		const poster = `https:${element.querySelector('img').src}`;
-		const teaser = sceneLinkElement.dataset.preview_clip_url;
-
-		const channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
-
-		return {
-			url,
-			entryId,
-			title,
-			actors,
-			poster,
-			teaser: {
-				src: teaser,
-			},
-			site,
-			channel,
-		};
-	}).filter(Boolean);
 }
 
-async function scrapeScene(html, url, site) {
+async function scrapeScene({ query }, url, channel, baseScene, parameters) {
-	const { document } = new JSDOM(html).window;
+	const release = {};
-
-	const title = document.querySelector('.description-title').textContent;
-	const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
-	const metaDescription = document.querySelector('meta[itemprop="description"]').content;
-	const description = metaDescription
-		? metaDescription.content
-		: document.querySelector('.description')
-			.textContent
-			.replace(/[ \t\n]{2,}/g, ' ')
-			.replace('...read more', '')
-			.trim();
-
-	const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
 	const { origin, pathname } = new URL(url);
-	const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
 
-	const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
+	release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase();
-	const duration = moment
+	release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`;
-		.duration(`00:${document
-			.querySelectorAll('.extra-info p')[1]
-			.textContent
-			.match(/\d+:\d+$/)[0]}`)
-		.asSeconds();
 
-	const trailerElement = document.querySelector('.html5-video');
+	release.title = query.cnt('.description-title') || query.text('.scene-title');
-	const poster = `https:${trailerElement.dataset.poster}`;
+	release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({
-	const { trailer } = trailerElement.dataset;
+		name: query.cnt(actorEl),
+		url: query.url(actorEl, null, 'href', { origin: channel.url }),
+	}));
 
-	const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0]?.href;
+	release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description, [itemprop="description"]')?.replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim();
-	const photos = lastPhotosUrl ? await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url) : [];
 
-	const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
+	release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content');
-	const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
+	release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container');
 
-	return {
+	release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/);
-		entryId,
+
-		url: `${origin}${pathname}`,
+	const trailer = query.video('.html5-video', 'data-trailer');
-		title,
+	const lastPhotosUrl = query.urls('.pagination a').at(-1);
-		description,
+
-		actors,
+	release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image');
-		date,
+
-		duration,
+	if (trailer && !trailer?.includes('join')) {
-		poster,
+		release.trailer = trailer;
-		photos,
-		trailer: {
-			src: trailer,
-		},
-		tags,
-		rating: {
-			stars,
-		},
-		site,
-		channel,
-	};
 	}
 
-async function fetchLatest(site, page = 1) {
+	if (lastPhotosUrl && parameters.includePhotos) {
-	const res = await http.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+		release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url);
-
-	return scrapeLatest(res.body.toString(), site);
 	}
 
-async function fetchScene(url, site) {
+	release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2));
-	const res = await http.get(url);
 
-	return scrapeScene(res.body.toString(), url, site);
+	return release;
+}
+
+async function fetchLatest(channel, page = 1) {
+	// const res = await qu.getAll(`https://dogfartnetwork.com/tour/scenes/?p=${page}`, '.recent-updates');
+	const res = await qu.getAll(`${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container');
+
+	if (res.ok) {
+		return scrapeLatest(res.items, channel);
+	}
+
+	return res.status;
 }
 
 async function fetchProfile(baseActor, entity) {
 	const slug = slugify(baseActor.name, '+');
 	const url = `https://www.dogfartnetwork.com/tour/girls/${slug}/`;
 
-	const res = await http.get(url);
+	const res = await qu.getAll(url, '.recent-updates');
 
 	if (res.ok) {
-		const scenes = scrapeLatest(res.body, entity, false);
+		const { scenes } = scrapeLatest(res.items, entity, false);
 
+		// no bio available
 		return { scenes };
 	}
 
@@ -156,6 +122,6 @@ async function fetchProfile(baseActor, entity) {
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
 	fetchProfile,
+	scrapeScene,
 };

src/scrapers/insex.js

@@ -5,6 +5,27 @@ const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
 function scrapeLatest(scenes, site) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+
+		release.title = query.cnt('.has-text-weight-bold, .is-size-6');
+		release.date = query.date('span.tag', 'YYYY-MM-DD');
+		release.actors = query.cnts('a.tag');
+
+		const cover = query.img('.image img');
+
+		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
+		release.covers = [cover];
+
+		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+
+		return release;
+	});
+}
+
+function scrapeLatestLegacy(scenes, site) {
 	return scenes.map(({ query }) => {
 		// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
 		const release = {};
@@ -47,28 +68,35 @@ function scrapeLatest(scenes, site) {
 	});
 }
 
-function scrapeLatestAlt(scenes, site) {
+async function scrapeScene({ query }, url, channel, parameters, session) {
-	return scenes.map(({ query }) => {
 	const release = {};
 
-		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+	release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
+	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
+	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
 
-		release.title = query.cnt('.has-text-weight-bold');
+	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
-		release.date = query.date('span.tag', 'YYYY-MM-DD');
+	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
-		release.actors = query.cnts('a.tag');
 
-		const cover = query.img('.image img');
+	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
-
+	release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
-		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
-		release.covers = [cover];
 
 	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
 
-		return release;
+	release.trailer = query.video();
-	});
+
+	if (!release.trailer && parameters.includeTrailers) {
+		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
+
+		if (trailerRes.ok) {
+			release.trailer = trailerRes.body;
+		}
 	}
 
-function scrapeScene({ query }, site) {
+	return release;
+}
+
+function scrapeSceneLegacy({ query }, site) {
 	const release = {};
 
 	const titleEl = query.q('.articleTitleText');
@@ -97,70 +125,34 @@ function scrapeScene({ query }, site) {
 	return release;
 }
 
-async function scrapeSceneAlt({ query }, url, channel, session) {
-	const release = {};
-
-	release.title = query.cnt('.columns div.is-size-5');
-	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
-	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
-
-	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
-	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
-
-	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
-	release.photos = query.imgs('body > div:nth-child(6) img');
-
-	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
-
-	release.trailer = query.video();
-
-	if (!release.trailer) {
-		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
-
-		if (trailerRes.ok) {
-			release.trailer = trailerRes.body;
-		}
-	}
-
-	return release;
-}
-
 async function fetchLatest(site, page = 1) {
-	const url = (site.parameters?.scraper === 'alt' && `${site.parameters.latest}/home.php?o=latest&p=${page}`)
+	const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
-		// || (site.slug === 'paintoy' && `${site.url}/corporal/punishment/gallery.php?type=brief&page=${page}`) // paintoy's site is (was?) partially broken, use front page
+	const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
-		|| `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-
-	const res = await ((site.parameters?.scraper === 'alt' && qu.getAll(url, 'body > .columns .column'))
-		// || (site.slug === 'paintoy' && qu.getAll(url, '#articleTable table[cellspacing="2"]'))
-		|| qu.get(url)); // JSON containing html as a property
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
-			return scrapeLatestAlt(res.items, site);
-		}
-
-		/*
-		if (site.slug === 'paintoy') {
 		return scrapeLatest(res.items, site);
 	}
-		*/
-
-		return scrapeLatest(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
-	}
 
 	return res.status;
 }
 
-async function fetchScene(url, site) {
+async function fetchLatestLegacy(site, page = 1) {
-	const session = http.session();
+	const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-	const res = await qu.get(url, null, null, { session });
+	const res = await qu.get(url); // JSON containing html as a property
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
+		return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
-			return scrapeSceneAlt(res.item, url, site, session);
 	}
 
-		return scrapeScene(res.item, site);
+	return res.status;
+}
+
+async function fetchScene(url, site, baseRelease, parameters) {
+	const session = http.session();
+	const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, site, parameters, session);
 	}
 
 	return res.status;
@@ -169,4 +161,8 @@ async function fetchScene(url, site) {
 module.exports = {
 	fetchLatest,
 	fetchScene,
+	legacy: {
+		fetchLatest: fetchLatestLegacy,
+		scrapeScene: scrapeSceneLegacy,
+	},
 };

src/scrapers/kellymadison.js

@@ -16,7 +16,7 @@ const siteMapByKey = {
 const siteMapBySlug = Object.entries(siteMapByKey).reduce((acc, [key, value]) => ({ ...acc, [value]: key }), {});
 
 function scrapeLatest(scenes, site) {
-	return scenes.map(({ query }) => {
+	return scenes.reduce((acc, { query }) => {
 		const release = {};
 
 		release.shootId = query.q('.card-meta .text-right, .row .text-right, .card-footer-item:last-child', true);
@@ -24,11 +24,6 @@ function scrapeLatest(scenes, site) {
 		const siteId = release.shootId.match(/\d?\w{2}/)[0];
 		const siteSlug = siteMapByKey[siteId];
 
-		if (site.slug !== siteSlug) {
-			// using generic network overview, scene is not from the site we want
-			return null;
-		}
-
 		const { pathname } = new URL(query.url('h5 a, .ep-title a, .title a'));
 		[release.entryId] = pathname.match(/\d+$/);
 		release.url = `${site.url}${pathname}`;
@@ -52,8 +47,16 @@ function scrapeLatest(scenes, site) {
 			};
 		}
 
-		return release;
+		if (site.slug !== siteSlug) {
-	}).filter((scene) => scene);
+			// using generic network overview, scene is not from the site we want
+			return { ...acc, unextracted: [...acc.unextracted, release] };
+		}
+
+		return { ...acc, scenes: [...acc.scenes, release] };
+	}, {
+		scenes: [],
+		unextracted: [],
+	});
 }
 
 async function scrapeScene({ query, html }, url, baseRelease, channel, session) {

src/scrapers/pervcity.js

@@ -12,6 +12,13 @@ const channelCodes = {
 	uha: 'upherasshole',
 };
 
+const qualities = {
+	v4k: 2160,
+	vFullHD: 1080,
+	vHD: 720,
+	vSD: 480,
+};
+
 const channelRegExp = new RegExp(Object.keys(channelCodes).join('|'), 'i');
 
 function scrapeAll(scenes, entity) {
@@ -42,9 +49,12 @@ function scrapeScene({ query }) {
 	release.entryId = query.q('.trailerLeft img', 'id').match(/set-target-(\d+)/)[1];
 
 	release.title = query.cnt('.infoHeader h1');
-	release.description = query.cnt('.infoBox p');
+	release.description = query.cnt('.description');
+	release.duration = query.duration('.tRuntime');
 
 	release.actors = query.cnts('.infoBox .tour_update_models a');
+	release.tags = query.cnts('.tagcats a');
+	release.qualities = query.imgs('.avaiFormate img').map((src) => qualities[src.match(/\/(\w+)\.png/)[1]]).filter(Boolean);
 
 	release.poster = query.img('.posterimg');
 	release.photos = query.imgs('.trailerSnaps img').slice(1); // first photo is poster in lower quality

src/store-releases.js

@@ -38,11 +38,8 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
 		date_precision: release.datePrecision,
 		slug,
 		description: release.description,
+		qualities: release.qualities?.map(Number).filter(Boolean),
 		comment: release.comment,
-		// director: release.director,
-		// likes: release.rating && release.rating.likes,
-		// dislikes: release.rating && release.rating.dislikes,
-		// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
 		deep: typeof release.deep === 'boolean' ? release.deep : false,
 		deep_url: release.deepUrl,
 		updated_batch_id: batchId,