1.256.5

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.255.19",
+    "version": "1.256.5",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.255.19",
+            "version": "1.256.5",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.255.19",
+    "version": "1.256.5",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/01_networks.js

@@ -172,7 +172,7 @@ const networks = [
 	{
 		slug: 'familystrokes',
 		name: 'Family Strokes',
-		url: 'https://www.familystrokes.com/',
+		url: 'https://www.familystrokes.com',
 		parent: 'paperstreetmedia',
 		parameters: {
 			endpoint: 'familybundle',

seeds/02_sites.js

@@ -8517,6 +8517,33 @@ const sites = [
 			id: 'hookup-pad',
 		},
 	},
+	{
+		slug: 'thepickupvan',
+		name: 'The Pickup Van',
+		url: 'https://www.mylf.com/series/the-pickup-van',
+		parent: 'mylf',
+		parameters: {
+			id: 'the-pickup-van',
+		},
+	},
+	{
+		slug: 'motherfucker',
+		name: 'Motherfucker',
+		url: 'https://www.mylf.com/series/motherfucker',
+		parent: 'mylf',
+		parameters: {
+			id: 'motherfucker',
+		},
+	},
+	{
+		slug: 'bluecollarbabes',
+		name: 'Blue Collar Babes',
+		url: 'https://www.mylf.com/series/blue-collar-babes',
+		parent: 'mylf',
+		parameters: {
+			id: 'blue-collar-babes',
+		},
+	},
 	/* premium */
 	{
 		slug: 'analmom',
@@ -8551,12 +8578,9 @@ const sites = [
 	{
 		slug: 'mylfdom',
 		name: 'MYLF Dom',
-		url: 'https://www.mylfdom.com',
+		url: 'https://www.mylf.com/series/mylfdom-ts',
 		tags: ['bdsm', 'milf'],
 		parent: 'mylf',
-		parameters: {
-			endpoint: 'mylf_dom',
-		},
 	},
 	// NAUGHTY AMERICA
 	{
@@ -12397,13 +12421,6 @@ const sites = [
 		url: 'https://www.teamskeet.com/series/bad-milfs',
 		parent: 'teamskeet',
 	},
-	{
-		slug: 'blackstepdad',
-		name: 'Black StepDad',
-		url: 'https://www.teamskeet.com/series/black-stepdad',
-		alias: ['bsd'],
-		parent: 'teamskeet',
-	},
 	{
 		slug: 'exxxtrasmall',
 		name: 'Exxxtra Small',
@@ -12783,6 +12800,44 @@ const sites = [
 		url: 'https://www.teamskeet.com/series/passport-bros',
 		parent: 'teamskeet',
 	},
+	{
+		slug: 'underthetable',
+		name: 'Under the Table',
+		url: 'https://www.teamskeet.com/series/under-the-table',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'mydirtyuncle',
+		name: 'My Dirty Uncle',
+		url: 'https://www.teamskeet.com/series/my-dirty-uncle',
+		tags: ['family'],
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'mormongirlz',
+		name: 'Mormon Girlz',
+		url: 'https://www.teamskeet.com/series/mormon-girlz',
+		tags: ['religion'],
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'breedingmaterial',
+		name: 'Breeding Material',
+		url: 'https://www.teamskeet.com/series/breeding-material',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'teamskeetsingles',
+		name: 'TeamSkeet Singles',
+		url: 'https://www.teamskeet.com/series/teamskeet-singles',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'sheseducedme',
+		name: 'She Seduced Me',
+		url: 'https://www.teamskeet.com/series/she-seduced-me',
+		parent: 'teamskeet',
+	},
 	// PERVZ
 	{
 		slug: 'pervzfeatures',
@@ -12859,6 +12914,12 @@ const sites = [
 		url: 'https://www.pervz.com/series/pervdriver',
 		parent: 'pervz',
 	},
+	{
+		slug: 'charmed',
+		name: 'Charmed',
+		url: 'https://www.pervz.com/series/charmed',
+		parent: 'pervz',
+	},
 	// SWAPPZ
 	{
 		slug: 'swappzfeatures',
@@ -12925,6 +12986,13 @@ const sites = [
 		parent: 'freeuse',
 		tags: ['free-use', 'pov'],
 	},
+	{
+		slug: 'freeusesingles',
+		name: 'Freeuse Singles',
+		url: 'https://www.freeuse.com/series/freeuse-singles',
+		parent: 'freeuse',
+		tags: ['free-use'],
+	},
 	// FAMILY STROKES
 	{
 		slug: 'familystrokes',
@@ -13006,6 +13074,14 @@ const sites = [
 		tags: ['family', 'milf'],
 		parent: 'familystrokes',
 	},
+	{
+		slug: 'blackstepdad',
+		name: 'Black StepDad',
+		url: 'https://www.familystrokes.com/series/black-stepdad',
+		alias: ['bsd'],
+		tags: ['family'],
+		parent: 'familystrokes',
+	},
 	// SAY UNCLE
 	{
 		slug: 'blackgodz',

src/deep.js

@@ -9,7 +9,6 @@ const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
 const { fetchReleaseEntities, urlToHostname } = require('./entities');
 const logger = require('./logger')(__filename);
-const qu = require('./utils/qu');
 const getRecursiveParameters = require('./utils/get-recursive-parameters');
 const windows = require('./utils/http-windows');
 
@@ -85,35 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
 	}
 
 	if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
-		/*
+		return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
-			return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		}
-		*/
-
-		if (!scraper.deprecated) {
-			return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		}
-
-		const session = qu.session();
-
-		const res = await qu.get(url, null, null, {
-			session,
-			rejectUnauthorized: false,
-		});
-
-		const cookie = await session._sessionOptions.cookieJar.get(url);
-
-		if (res.ok) {
-			return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
-				session,
-				headers: res.headers,
-				cookieJar: session._sessionOptions.cookieJar,
-				cookie,
-			});
-		}
-
-		return res.status;
 	}
 
 	return null;

src/media.js

@@ -15,6 +15,7 @@ const sharp = require('sharp');
 const blake2 = require('blake2');
 const { format } = require('date-fns');
 const taskQueue = require('promise-task-queue');
+const unprint = require('unprint');
 
 const { Upload } = require('@aws-sdk/lib-storage');
 const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
@@ -25,7 +26,6 @@ const knex = require('./knex');
 const http = require('./utils/http');
 const batchInsert = require('./utils/batch-insert');
 const chunk = require('./utils/chunk');
-const { get } = require('./utils/qu');
 const { fetchEntityReleaseIds } = require('./entity-releases');
 
 const streamQueue = taskQueue();
@@ -383,10 +383,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 		};
 	}
 
-	const res = await get(baseSource.url);
+	const res = await unprint.get(baseSource.url);
 
 	if (res.ok) {
-		const src = await baseSource.extract(res.item);
+		const src = await baseSource.extract(res.context);
 
 		return {
 			...baseSource,

src/scrapers/actors.js

@@ -29,7 +29,6 @@ const karups = require('./karups');
 const kellymadison = require('./kellymadison');
 const kink = require('./kink');
 const mariskax = require('./mariskax');
-// const analvids = require('./analvids');
 const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
 const testedefudelidade = require('./testedefudelidade');
 const theflourish = require('./theflourish');
 const tokyohot = require('./tokyohot');
-// const topwebmodels = require('./topwebmodels');
 const virtualtaboo = require('./virtualtaboo');
 const vixen = require('./vixen');
 const wankzvr = require('./wankzvr');
@@ -229,7 +227,6 @@ module.exports = {
 	boyfun: karups,
 	kellymadison,
 	'8kmembers': kellymadison,
-	// analvids,
 	analvids: pornbox,
 	littlecapricedreams,
 	mariskax,

src/scrapers/americanpornstar.js

@@ -1,45 +1,45 @@
 'use strict';
 
-const { fetchLatest, fetchProfile } = require('./julesjordan');
+const unprint = require('unprint');
 
-const qu = require('../utils/q');
+const { fetchLatest, fetchProfile } = require('./julesjordan');
 const slugify = require('../utils/slugify');
 
 function scrapeScene({ query }, channel) {
 	const release = {};
-	const title = query.cnt('.update_title', false);
+	const title = query.content('.update_title', { trim: false });
 
 	release.title = title.trim();
-	release.description = query.cnt('.latest_update_description');
+	release.description = query.content('.latest_update_description');
 
 	release.date = query.date('.update_date', 'MM/DD/YYYY');
 	release.actors = query.all('.tour_update_models a').map((actorEl) => ({
-		name: query.cnt(actorEl),
+		name: unprint.query.content(actorEl),
-		url: query.url(actorEl, null),
+		url: unprint.query.url(actorEl, null),
 	}));
 
 	release.entryId = slugify(release.title);
 
-	release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
+	release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
 
-	const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
+	const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
-	const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
+	const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
 
 	release.poster = [
 		poster.replace(/.jpg$/, '-full.jpg'),
 		poster,
 	];
 
-	release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
+	release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
 		img.replace(/.jpg$/, '-full.jpg'),
 		img,
 	]);
 
 	if (trailer) {
-		release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
+		release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
 	}
 
-	release.tags = query.cnts('.tour_update_tags a');
+	release.tags = query.contents('.tour_update_tags a');
 
 	return release;
 }
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel);
+		return scrapeScene(res.context, channel);
 	}
 
 	return res.status;

src/scrapers/analvids.js

@@ -1,174 +0,0 @@
-'use strict';
-
-const unprint = require('unprint');
-
-const slugify = require('../utils/slugify');
-
-function extractTitle(originalTitle) {
-	const titleComponents = originalTitle.split(' ');
-	// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
-	const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
-	const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
-	const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
-
-	return { shootId, title };
-}
-
-function scrapeAll(scenes, channel) {
-	return scenes.map(({ query }) => {
-		const release = {};
-
-		release.url = query.url('.card-scene__view > a');
-		release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
-
-		release.title = query.content('.card-scene__text');
-		release.shootId = extractTitle(release.title).shootId;
-
-		release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
-		release.duration = query.duration('.label--time:first-child');
-
-		release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
-		const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
-
-		if (caps && release.poster) {
-			release.caps = caps;
-		} else if (caps) {
-			[release.poster, ...release.caps] = caps;
-		}
-
-		release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
-
-		return release;
-	});
-}
-
-async function fetchLatest(channel, page) {
-	// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
-	// studios as channels
-	const url = `${channel.url}/latest/${page}`;
-
-	const res = await unprint.get(url, {
-		selectAll: '.card-scene',
-		headers: {
-			Referer: url,
-		},
-	});
-
-	if (res.ok) {
-		return scrapeAll(res.context, channel);
-	}
-
-	return res.status;
-}
-
-function scrapeScene({ query }, url) {
-	const release = {};
-
-	release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
-
-	const featuring = query.content('.watch__title .watch__featuring_models');
-
-	release.title = query.content('.watch__title').replace(featuring, '');
-	release.description = query.content('.text-mob-more');
-	release.shootId = extractTitle(release.title).shootId;
-
-	release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
-	release.duration = query.duration('.bi-clock');
-
-	release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
-		name: unprint.query.content(el),
-		url: unprint.query.url(el, null),
-	}));
-
-	release.tags = query.contents('.genres-list a[href*="/genre"]');
-
-	// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
-
-	release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
-
-	release.trailer = query.all('.watch__video source').map((el) => ({
-		src: unprint.query.video(el, null, { attribute: 'src' }),
-		quality: unprint.query.number(el, null, { attribute: 'size' }),
-	}));
-
-	return release;
-}
-
-async function fetchScene(url) {
-	const res = await unprint.get(url, {
-		headers: {
-			Referer: url,
-		},
-	});
-
-	if (res.ok) {
-		return scrapeScene(res.context, url);
-	}
-
-	return res.status;
-}
-
-function scrapeProfile({ query }, url, channel) {
-	const profile = { url };
-
-	profile.nationality = query.content('.model__info a[href*="/nationality"]');
-	profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
-
-	profile.avatar = query.img('.model__left img');
-
-	profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
-
-	return profile;
-}
-
-async function getActorUrl(actor, channel) {
-	if (actor.url) {
-		return actor.url;
-	}
-
-	const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
-
-	const searchRes = await unprint.get(searchUrl, {
-		headers: {
-			Referer: actor.url,
-		},
-	});
-
-	if (!searchRes.ok) {
-		return searchRes.status;
-	}
-
-	const result = searchRes.body.terms.find((item) => item.type === 'model');
-
-	if (result) {
-		return result.url;
-	}
-
-	return null;
-}
-
-async function fetchProfile(actor, { channel }) {
-	const actorUrl = await getActorUrl(actor, channel);
-
-	if (typeof actorUrl !== 'string') {
-		return actorUrl;
-	}
-
-	const bioRes = await unprint.get(actorUrl, {
-		headers: {
-			Referer: actor.url,
-		},
-	});
-
-	if (bioRes.ok) {
-		return scrapeProfile(bioRes.context, actorUrl, channel);
-	}
-
-	return bioRes.status;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchScene,
-	fetchProfile,
-};

src/scrapers/analvids_legacy.js

@@ -1,203 +0,0 @@
-'use strict';
-
-const { JSDOM } = require('jsdom');
-const cheerio = require('cheerio');
-const moment = require('moment');
-
-const http = require('../utils/http');
-const slugify = require('../utils/slugify');
-
-function extractTitle(originalTitle) {
-	const titleComponents = originalTitle.split(' ');
-	// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
-	const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
-	const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
-	const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
-
-	return { shootId, title };
-}
-
-function getPoster(posterElement, sceneId) {
-	const posterStyle = posterElement.attr('style');
-
-	if (posterStyle) {
-		return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
-	}
-
-	const posterRange = posterElement.attr('data-casting');
-	const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
-	const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
-
-	if (!posterTimeRange) {
-		return null;
-	}
-
-	if (typeof posterTimeRange === 'number') {
-		// poster time is already a single time value
-		return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
-	}
-
-	const [max, min] = posterTimeRange.split('-');
-	const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
-
-	return `https://analvids.com/casting/${sceneId}/${posterTime}`;
-}
-
-function scrapeAll(html) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
-	const scenesElements = $('.thumbnails > div').toArray();
-
-	return scenesElements.map((element) => {
-		const sceneLinkElement = $(element).find('.thumbnail-title a');
-		const url = sceneLinkElement.attr('href');
-
-		const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
-		const { shootId, title } = extractTitle(originalTitle);
-		const entryId = new URL(url).pathname.split('/')[2];
-
-		const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
-
-		const sceneId = $(element).attr('data-content');
-		const posterElement = $(element).find('.thumbnail-avatar');
-
-		const poster = getPoster(posterElement, sceneId);
-
-		return {
-			url,
-			shootId,
-			entryId,
-			title,
-			date,
-			poster,
-		};
-	});
-}
-
-async function scrapeScene(html, url, site, useGallery) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
-	const playerObject = $('script:contains("new WatchPage")').html();
-	const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
-	const data = playerData && JSON.parse(playerData);
-
-	const release = { url };
-
-	const originalTitle = $('h1.watchpage-title').text().trim();
-	const { shootId, title } = extractTitle(originalTitle);
-
-	release.shootId = shootId;
-	release.entryId = new URL(url).pathname.split('/')[2];
-
-	release.title = title;
-	release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
-
-	const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
-
-	release.description = $('meta[name="description"]')?.attr('content')?.trim()
-        || (descriptionElement && $(descriptionElement).find('dd').text().trim());
-
-	release.actors = $(actorsElement)
-		.find('a[href*="com/model"]')
-		.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
-
-	release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
-	release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-
-	const photos = useGallery
-		? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
-		: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
-
-	release.photos = photos.map((source) => {
-		// source without parameters sometimes serves larger preview photo
-		const { origin, pathname } = new URL(source);
-
-		return `${origin}${pathname}`;
-	});
-
-	const posterStyle = $('#player').attr('style');
-	const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
-
-	release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
-
-	if (data) {
-		const qualityMap = {
-			web: 240,
-			vga: 480,
-			hd: 720,
-			'1080p': 1080,
-		};
-
-		release.trailer = data.clip.qualities.map((trailer) => ({
-			src: trailer.src,
-			type: trailer.type,
-			quality: qualityMap[trailer.quality] || trailer.quality,
-		}));
-	}
-
-	const studioName = $('.watchpage-studioname').first().text().trim();
-	release.studio = slugify(studioName, '', { removePunctuation: true });
-
-	return release;
-}
-
-async function scrapeProfile(html, _url, actorName) {
-	const { document } = new JSDOM(html).window;
-
-	const profile = {
-		name: actorName,
-	};
-
-	const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
-	const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
-
-	const bio = entries
-		.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
-		.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
-
-	profile.birthPlace = bio.Nationality;
-
-	if (bio.Age) profile.age = bio.Age;
-	if (avatarEl) profile.avatar = avatarEl.src;
-
-	profile.releases = scrapeAll(html);
-
-	return profile;
-}
-
-async function fetchLatest(site, page = 1) {
-	const res = await http.get(`${site.url}/new-videos/${page}`);
-
-	return scrapeAll(res.body.toString(), site);
-}
-
-async function fetchScene(url, site) {
-	const useGallery = true;
-
-	// TODO: fall back on screenshots when gallery is not available
-	const res = useGallery
-		? await http.get(`${url}/gallery#gallery`)
-		: await http.get(`${url}/screenshots#screenshots`);
-
-	return scrapeScene(res.body.toString(), url, site, useGallery);
-}
-
-async function fetchProfile({ name: actorName }) {
-	const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
-	const data = res.body;
-
-	const result = data.terms.find((item) => item.type === 'model');
-
-	if (result) {
-		const bioRes = await http.get(result.url);
-		const html = bioRes.body.toString();
-
-		return scrapeProfile(html, result.url, actorName);
-	}
-
-	return null;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchProfile,
-	fetchScene,
-};

src/scrapers/assylum.js

@@ -1,6 +1,9 @@
 'use strict';
 
-const { get, geta, ctxa } = require('../utils/q');
+const unprint = require('unprint');
+
+// paths are relative, read raw attributes instead of resolved URLs
+const raw = { forceGetAttribute: true };
 
 function extractActors(actorString) {
 	return actorString
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
 }
 
 function scrapeLatest(scenes, site, models) {
-	return scenes.map(({ qu }) => {
+	return scenes.map(({ query }) => {
 		const release = {};
 
-		const pathname = qu.url('a.itemimg').slice(1);
+		const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
 		[release.entryId] = pathname.split('/').slice(-1);
 		release.url = `${site.url}${pathname}`;
 
-		release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
+		release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
-		release.description = qu.q('.mas_longdescription', true);
+		release.description = query.content('.mas_longdescription');
-		release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
+		release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
 
-		const actorString = qu.q('.mas_description', true);
+		const actorString = query.content('.mas_description');
 		const actors = matchActors(actorString, models);
 
 		if (actors.length > 0) release.actors = actors;
 		else release.actors = extractActors(actorString);
 
-		const posterPath = qu.img('.itemimg img');
+		const posterPath = query.attribute('.itemimg img', 'src', raw);
 		release.poster = `${site.url}/${posterPath}`;
 
 		return release;
 	});
 }
 
-function scrapeScene({ html, qu }, url, site, include, models) {
+function scrapeScene({ html, query }, url, site, include, models) {
 	const release = { url };
 
 	[release.entryId] = url.split('/').slice(-1);
-	release.title = qu.q('.mas_title', true);
+	release.title = query.content('.mas_title');
-	release.description = qu.q('.mas_longdescription', true);
+	release.description = query.content('.mas_longdescription');
-	release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
 
 	if (models) {
-		const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
+		const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
 		const actors = matchActors(actorString, models);
 		if (actors.length > 0) release.actors = actors;
 		else release.actors = extractActors(actorString);
 	}
 
-	release.tags = qu.all('.tags a', true);
+	release.tags = query.contents('.tags a');
 
-	release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
+	release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
 
 	const posterIndex = 'splash:';
 	const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
 	return release;
 }
 
-function extractModels({ el }, site) {
+function extractModels({ element }, site) {
-	const models = ctxa(el, '.item');
+	const models = unprint.initAll(element, '.item');
 
-	return models.map(({ qu }) => {
+	return models.map(({ query }) => {
 		const actor = { gender: 'female' };
 
-		const avatar = qu.q('.itemimg img');
+		const avatar = query.element('.itemimg img');
 		actor.avatar = `${site.url}/${avatar.src}`;
 		actor.name = avatar.alt
 			.split(':').slice(-1)[0]
 			.replace(/xtreme girl|nurse/ig, '')
 			.trim();
 
-		const actorPath = qu.url('.itemimg');
+		const actorPath = query.attribute('.itemimg', 'href', raw);
 		actor.url = `${site.url}${actorPath.slice(1)}`;
 
 		return actor;
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
 
 async function fetchModels(site, page = 1, accModels = []) {
 	const url = `${site.url}/?models/${page}`;
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const models = extractModels(res.item, site);
+		const models = extractModels(res.context, site);
-		const nextPage = res.item.qa('.pagenumbers', true)
+		const nextPage = res.context.query.contents('.pagenumbers')
 			.map((pageX) => Number(pageX))
 			.filter(Boolean) // remove << and >>
 			.includes(page + 1);
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
 
 async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
 	const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
-	const res = await geta(url, '.item');
+	const res = await unprint.get(url, { selectAll: '.item' });
 
-	return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
+	return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
 }
 
 async function fetchScene(url, site, release, include, beforeFetchLatest) {
 	const models = beforeFetchLatest || await fetchModels(site);
-	const res = await get(url);
+	const res = await unprint.get(url);
 
-	return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
+	return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
 }
 
 module.exports = {

src/scrapers/badoink.js

@@ -1,6 +1,7 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
+
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('a', 'href', { origin: channel.url });
+		release.url = query.url('a', { origin: channel.url });
-		release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
+		release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
 
-		release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
+		release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
-		release.description = query.cnt('.video-card-description');
+		release.description = query.content('.video-card-description');
 
-		release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
+		release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
-		release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
+		release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
 
 		release.actors = query.all('.video-card-details--cast a').map((el) => ({
-			name: qu.query.cnt(el),
+			name: unprint.query.content(el),
-			url: qu.query.url(el, null, 'href', { origin: channel.url }),
+			url: unprint.query.url(el, null, { origin: channel.url }),
 		}));
 
-		release.tags = query.cnts('.video-card-tags a');
+		release.tags = query.contents('.video-card-tags a');
 
-		const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
+		const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
 		release.poster = getPoster(posterSources);
 
 		release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
 	});
 }
 
-function scrapeScene({ query }, url, channel) {
+function scrapeScene({ query }, { url, entity: channel }) {
 	const release = {};
 
 	release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
 
-	release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
+	release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
-	release.description = query.cnt('.video-description');
+	release.description = query.content('.video-description');
 
-	release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
-	release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
+	release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
 
 	release.actors = query.all('.video-actors a').map((el) => ({
-		name: qu.query.cnt(el),
+		name: unprint.query.content(el),
-		url: qu.query.url(el, null, 'href', { origin: channel.url }),
+		url: unprint.query.url(el, null, { origin: channel.url }),
 	}));
 
-	release.tags = query.cnts('.video-tags a');
+	release.tags = query.contents('.video-tags a');
 
-	const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
+	const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
 	release.poster = getPoster(posterSources);
 
-	const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
+	const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
-	const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
+	const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
 	const lastPhoto = photos.slice(-1)[0];
 	const photosHaveIndex = /_\d+\./.test(lastPhoto);
 
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
 
 	const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
 		...acc,
-		[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
+		[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
 	}), {});
 
-	profile.description = query.cnt('.girl-details-bio');
+	profile.description = query.content('.girl-details-bio');
 
 	profile.age = bio.age;
 	profile.birthPlace = bio.country;
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
 	profile.hairColor = bio.hair;
 	profile.eyes = bio.eyes;
 
-	const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
+	const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
 
 	profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
 	profile.social = query.urls('.girl-details-social-media-list a');
 
-	profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
+	profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
 
 	return profile;
 }
 
 async function fetchLatest(channel, page) {
-	const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
+	const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
-		Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
+		selectAll: '.video-card',
+		cookies: {
+			affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
+		},
 	});
 
 	if (res.ok) {
-		return scrapeAll(res.items, channel);
+		return scrapeAll(res.context, channel);
 	}
 
 	return res.status;
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
 
 async function fetchProfile(baseActor, { entity }) {
 	const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeProfile(res.item, url, entity);
+		return scrapeProfile(res.context, url, entity);
 	}
 
 	return res.status;

src/scrapers/czechav.js

@@ -1,29 +1,29 @@
 'use strict';
 
-const qu = require('../utils/q');
+const unprint = require('unprint');
 
 function scrapeAll(scenes, channel) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('a', 'href', { origin: channel.url });
+		release.url = query.url('a', { origin: channel.url });
 		release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
 
-		release.title = query.cnt('.nice-title');
+		release.title = query.content('.nice-title');
 
 		release.poster = query.img('img.poster');
-		release.photos = query.q('img.poster', 'data-urls')?.split(' ');
+		release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
-		release.teaser = query.video('img.poster', 'data-thumbnail');
+		release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
 
 		if (query.exists('.info')) {
 			release.date = query.date('.updated', 'MM/DD/YYYY');
 
 			release.actors = [{
-				name: query.cnt('.name'),
+				name: query.content('.name'),
-				age: query.cnt('.age'),
+				age: query.content('.age'),
 			}];
 
-			release.tags = query.cnts('.tags a');
+			release.tags = query.contents('.tags a');
 		}
 
 		return release;
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
 
 	release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
 
-	release.title = query.cnt('.nice-title');
+	release.title = query.content('.nice-title');
-	release.description = query.cnt('.desc-text p');
+	release.description = query.content('.desc-text p');
 
-	release.tags = query.cnts('.tags a');
+	release.tags = query.contents('.tags a');
 
 	release.poster = query.poster();
 	release.photos = query.imgs('.thumbnails img, .thumb img');
 
 	// some videos may not have unique trailers, but many do
-	release.trailer = query.video();
+	release.trailer = query.video('source');
 
-	release.formats = query.cnts('.download-quality option');
+	release.formats = query.contents('.download-quality option');
 
 	return release;
 }
 
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/tour/videos/page-${page}`;
-	const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
+	const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
 
 	if (res.ok) {
-		return scrapeAll(res.items, channel);
+		return scrapeAll(res.context, channel);
 	}
 
 	return res.status;
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, channel);
+		return scrapeScene(res.context, url, channel);
 	}
 
 	return res.status;

src/scrapers/elevatedx.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const format = require('template-format');
+const unprint = require('unprint');
 
-const qu = require('../utils/q');
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
 	if (release.date && release.url) {
 		const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
 
-		return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
+		return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
 	}
 
 	if (release.date && release.title) {
-		return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
+		return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
 	}
 
 	return null;
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
 
 function extractPoster(posterPath, channel, baseRelease) {
 	if (posterPath && !/400.jpg/.test(posterPath)) {
-		const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
+		const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
 		const posterSources = [
 			poster,
 			// upscaled
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
 	return [baseRelease?.poster || null, []];
 }
 
-function getImageWithFallbacks(q, selector, site, el) {
+function getImageWithFallbacks(query, selector, site, el) {
 	const sources = el
 		? [
-			q(el, selector, 'src0_3x'),
+			unprint.query.attribute(el, selector, 'src0_3x'),
-			q(el, selector, 'src0_2x'),
+			unprint.query.attribute(el, selector, 'src0_2x'),
-			q(el, selector, 'src0_1x'),
+			unprint.query.attribute(el, selector, 'src0_1x'),
 		]
 		: [
-			q(selector, 'src0_3x'),
+			query.attribute(selector, 'src0_3x'),
-			q(selector, 'src0_2x'),
+			query.attribute(selector, 'src0_2x'),
-			q(selector, 'src0_1x'),
+			query.attribute(selector, 'src0_1x'),
 		];
 
-	return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
+	return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
 }
 
 function scrapeAllClassic(scenes, channel) {
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
 		const release = {};
 
 		release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
-		release.entryId = query.url('.updateThumb img', 'alt');
+		release.entryId = query.attribute('.updateThumb img', 'alt');
 
-		release.title = query.cnt('.updateInfo h5 a');
+		release.title = query.content('.updateInfo h5 a');
 
-		release.actors = query.cnts('.tour_update_models a');
+		release.actors = query.contents('.tour_update_models a');
 		release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
 
-		release.poster = query.img('.updateThumb img');
+		release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
 
-		const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
+		const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
 
 		if (trailer) {
 			release.trailer = `${channel.url}${trailer}`;
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
+		release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
 		release.url = query.url('h4 a');
 
 		release.date = query.date('.more-info-div', 'MMM D, YYYY');
-		release.duration = query.dur('.more-info-div');
+		release.duration = query.duration('.more-info-div');
 
-		const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
+		// raw path, prefixed with media URL below
+		const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
 
 		if (posterPath) {
 			const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 			];
 		}
 
-		release.teaser = query.video();
+		release.teaser = query.video('source');
 
 		// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
 		release.entryId = deriveEntryId(release);
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 	});
 }
 
-function scrapeSceneClassic({ query, html }, url, channel) {
+function scrapeSceneClassic({ query, html }, { entity: channel }) {
 	const release = {};
 
-	release.title = query.q('.updatesBlock h2', true);
+	release.title = query.content('.updatesBlock h2');
-	release.poster = query.meta('property="og:image"');
+	release.poster = query.attribute('meta[property="og:image"]', 'content');
 	release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
 
 	const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
 	return release;
 }
 
-function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
+function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
 	const release = {};
 
-	release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
+	release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
 	release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
 
-	release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
-	release.duration = query.dur('.update-info-row:nth-child(2)');
+	release.duration = query.duration('.update-info-row:nth-child(2)');
-	release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
+	release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
 
 	release.actors = query.all('.models-list-thumbs a').map((el) => ({
-		name: query.cnt(el, 'span'),
+		name: unprint.query.content(el, 'span'),
-		avatar: getImageWithFallbacks(query.q, 'img', entity, el),
+		avatar: getImageWithFallbacks(query, 'img', entity, el),
-		url: query.url(el, null),
+		url: unprint.query.url(el, null),
 	}));
 
-	release.tags = query.all('.tags a', true);
+	release.tags = query.contents('.tags a');
 
-	const posterPath = query.q('.player-thumb img', 'src0_1x');
+	const posterPath = query.attribute('.player-thumb img', 'src0_1x');
 	const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
 
 	[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
 
 	if (trailer) {
-		release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
+		release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
 	}
 
-	const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
+	const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
 	if (stars) release.stars = Number(stars);
 
 	if (entity.type === 'network') {
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
 async function scrapeProfile({ query }, entity, parameters) {
 	const profile = {};
 
-	const bio = query.cnt('.model_bio, .detail-div');
+	const bio = query.content('.model_bio, .detail-div');
-	const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
+	const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
 
 	profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
-	profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
+	profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
 
 	profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
 	profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
 			avatarEl.getAttribute('src'),
 		]
 			.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
-			.map((avatar) => qu.prefixUrl(avatar, entity.url));
+			.map((avatar) => unprint.prefixUrl(avatar, entity.url));
 
 		if (avatarSources.length) profile.avatar = avatarSources;
 	}
 
 	if (parameters?.layout === 'classic') {
-		profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
+		profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
 	}
 
 	if (parameters?.layout === 'tubular') {
-		profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
+		profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
 	}
 
 	return profile;
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
 	const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
         || `${site.url}/categories/movies_${page}_d.html`;
 
-	const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
+	const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
 
 	if (!res.ok) {
 		return res.status;
 	}
 
-	return allScraper(res.items, site, preData?.uniqueReleases);
+	return allScraper(res.context, site, preData?.uniqueReleases);
 }
 
 async function fetchUpcomingClassic(channel) {
-	const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
+	const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
 
 	if (res.ok) {
-		return scrapeAllClassic(res.items, channel);
+		return scrapeAllClassic(res.context, channel);
 	}
 
 	return res.status;
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
 			return null;
 		}
 
-		const res = await qu.get(profileUrl);
+		const res = await unprint.get(profileUrl);
 
-		if (res.statusCode === 200) {
+		if (res.status === 200) {
-			return scrapeProfile(res.item, entity, parameters);
+			return scrapeProfile(res.context, entity, parameters);
 		}
 
 		return null;
@@ -290,14 +291,12 @@ module.exports = {
 		fetchProfile,
 		scrapeAll: scrapeAllClassic,
 		scrapeScene: scrapeSceneClassic,
-		deprecated: true,
 	},
 	tubular: {
 		fetchLatest: fetchLatestTubular,
 		fetchProfile,
 		scrapeAll: scrapeAllTubular,
 		scrapeScene: scrapeSceneTubular,
-		deprecated: true,
 	},
 	getImageWithFallbacks,
 };

src/scrapers/insex.js

@@ -1,25 +1,25 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
-const http = require('../utils/http');
+
 const slugify = require('../utils/slugify');
 
 function scrapeLatest(scenes, site) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+		release.url = query.url('figure a', { origin: site.parameters.latest });
 
-		release.title = query.cnt('.has-text-weight-bold, .is-size-6');
+		release.title = query.content('.has-text-weight-bold, .is-size-6');
 		release.date = query.date('span.tag', 'YYYY-MM-DD');
-		release.actors = query.cnts('a.tag');
+		release.actors = query.contents('a.tag');
 
 		const cover = query.img('.image img');
 
 		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
 		release.covers = [cover];
 
-		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+		release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
 
 		return release;
 	});
@@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) {
 		// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
 		const release = {};
 
-		const titleEl = query.q('.galleryTitleText, .articleTitleText');
+		const titleEl = query.element('.galleryTitleText, .articleTitleText');
 		const [title, ...actors] = titleEl.textContent.split('|');
 		const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
 
-		const url = query.url(titleEl, 'a');
+		const url = unprint.query.attribute(titleEl, 'a', 'href');
 		[release.entryId] = url.split('/').slice(-2);
 		release.url = `${site.url}${url}`;
 
@@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) {
 		} else {
 			//  title should contain date instead, not applicable in brief mode
 			release.title = title.slice(title.indexOf(':') + 1).trim();
-			release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
+			release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
 		}
 
 		release.actors = actors.map((actor) => actor.trim());
 
-		const description = query.q('.articleCopyText', true);
+		const description = query.content('.articleCopyText');
 		if (description) release.description = description.slice(0, description.lastIndexOf('('));
 
-		const duration = query.dur('.articleCopyText a:nth-child(2)');
+		const duration = query.duration('.articleCopyText a:nth-child(2)');
 		if (duration) release.duration = duration;
 
-		release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
+		release.likes = query.number('.articlePostDateText td:nth-child(3)');
 
-		const cover = query.img('a img');
+		const cover = query.img('a img', { origin: site.url });
 
 		release.covers = [[
 			cover.replace('_thumbnail', ''),
@@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) {
 	});
 }
 
-async function scrapeScene({ query }, url, channel, parameters, session) {
+async function scrapeScene({ query }, channel, include, cookies) {
 	const release = {};
 
-	release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
+	release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
-	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
+	release.description = query.content('.has-background-black-ter > div:nth-child(4)');
 	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
 
-	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
+	release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
-	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
+	release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
 
-	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
+	release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
-	release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
+	release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
 
-	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
+	release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
 
-	release.trailer = query.video();
+	release.trailer = query.video('source');
 
-	if (!release.trailer && parameters.includeTrailers) {
+	if (!release.trailer && include.includeTrailers) {
-		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
+		// play API relies on the session cookie set by the scene page
+		const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
 
 		if (trailerRes.ok) {
 			release.trailer = trailerRes.body;
@@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
 	return release;
 }
 
-function scrapeSceneLegacy({ query }, site) {
+function scrapeSceneLegacy({ query }, { entity: site }) {
 	const release = {};
 
-	const titleEl = query.q('.articleTitleText');
+	const titleEl = query.element('.articleTitleText');
 	const [title, ...actors] = titleEl.textContent.split('|');
 
-	const url = query.url(titleEl, 'a');
+	const url = unprint.query.attribute(titleEl, 'a', 'href');
 	[release.entryId] = url.split('/').slice(-2);
 	release.url = `${site.url}${url}`;
 
 	release.title = title.trim();
-	release.description = query.q('.articleCopyText', true);
+	release.description = query.content('.articleCopyText');
 
 	release.actors = actors.map((actor) => actor.trim());
 	release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
-	release.duration = query.dur('.articlePostDateText a:nth-child(2)');
+	release.duration = query.duration('.articlePostDateText a:nth-child(2)');
 
 	const [cover, ...photos] = query.imgs('img[src*="images"]');
 	release.covers = [cover];
@@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) {
 
 	release.poster = query.poster();
 
-	const trailer = query.trailer();
+	const trailer = query.video('source');
 	if (trailer) release.trailer = { src: trailer };
 
 	return release;
@@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) {
 
 async function fetchLatest(site, page = 1) {
 	const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
-	const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
+	const res = await unprint.get(url, {
+		selectAll: 'body > .columns .column',
+		cookies: { consent: 'yes' },
+	});
 
 	if (res.ok) {
-		return scrapeLatest(res.items, site);
+		return scrapeLatest(res.context, site);
 	}
 
 	return res.status;
@@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) {
 
 async function fetchLatestLegacy(site, page = 1) {
 	const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-	const res = await qu.get(url); // JSON containing html as a property
+	const res = await unprint.get(url); // JSON containing html as a property
 
-	if (res.ok) {
+	if (res.ok && res.data?.html) {
-		return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
+		return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
 	}
 
 	return res.status;
 }
 
-async function fetchScene(url, site, baseRelease, parameters) {
+async function fetchScene(url, site, baseRelease, include) {
-	const session = http.session();
+	const res = await unprint.get(url, {
-	const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
+		cookies: { consent: 'yes' },
+	});
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, site, parameters, session);
+		return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
 	}
 
 	return res.status;

src/scrapers/jesseloadsmonsterfacials.js

@@ -1,11 +1,14 @@
 'use strict';
 
-const { get, initAll } = require('../utils/qu');
+const unprint = require('unprint');
+
+// paths are relative to /visitors/, read raw attributes instead of resolved URLs
+const raw = { forceGetAttribute: true };
 
 function scrapeLatest(scenes, dates, site) {
-	return scenes.map(({ qu }, index) => {
+	return scenes.map(({ query }, index) => {
 		const release = {};
-		const path = qu.url('a[href*="videos/"]');
+		const path = query.attribute('a[href*="videos/"]', 'href', raw);
 
 		if (path) {
 			if (/\.wmv$/.test(path)) {
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
 		}
 
 		if (dates && dates[index]) {
-			release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
+			release.date = dates[index].query.date(null, 'MM/DD/YYYY');
 		}
 
 		// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
 		release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
-			|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
+			|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
 
-		release.description = qu.q('tbody tr:nth-child(3) font', true);
+		release.description = query.content('tbody tr:nth-child(3) font');
 
-		const infoLine = qu.q('font[color="#663366"]', true);
+		const infoLine = query.content('font[color="#663366"]');
 
 		if (infoLine) {
 			release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
 		}
 
-		const poster = qu.img('img[src*="photos/"][width="400"]');
+		const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
 		release.poster = `${site.url}/visitors/${poster}`;
-		release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
+		release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
 
 		return release;
 	});
 }
 
-function scrapeScene({ qu }, url, site) {
+function scrapeScene({ query }, url, site) {
 	const release = { url };
 
 	const { pathname } = new URL(url);
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
 		return release;
 	}
 
-	const actor = qu.q('font[color="#990033"] strong', true);
+	const actor = query.content('font[color="#990033"] strong');
 	release.actors = [actor];
 
-	const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
+	const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
-	const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
+	const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
 
 	release.trailer = [
-		{
+		hdTrailer && {
 			src: `${site.url}/visitors/videos/${hdTrailer}`,
 			quality: 1080,
 		},
-		{
+		sdTrailer && {
 			src: `${site.url}/visitors/videos/${sdTrailer}`,
 			quality: 270,
 		},
-	];
+	].filter(Boolean);
 
 	return release;
 }
 
 async function fetchLatest(site, page = 1) {
 	const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (!res.ok) {
 		return res.status;
 	}
 
-	const { el } = res.item;
+	const { element } = res.context;
 
-	const scenes = initAll(el, 'table[width="880"]');
+	const scenes = unprint.initAll(element, 'table[width="880"]');
-	const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
+	const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
 
 	return scrapeLatest(scenes, dates, site);
 }
 
 async function fetchScene(url, site) {
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, site);
+		return scrapeScene(res.context, url, site);
 	}
 
 	return res.status;

src/scrapers/perfectgonzo.js

@@ -2,7 +2,7 @@
 
 const blake2 = require('blake2');
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
 
 function getHash(identifier) {
 	const hash = blake2.createHash('blake2b', { digestLength: 8 });
@@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.title = query.q('a', 'title');
+		release.title = query.attribute('a', 'title');
-		release.url = query.url('a', 'href', { origin: site.url });
+		release.url = query.url('a', { origin: site.url });
 		release.date = query.date('.nm-date', 'MM/DD/YYYY');
 
 		const slug = new URL(release.url).pathname.split('/')[2];
@@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) {
 
 		[release.poster, ...release.photos] = query.imgs('.bloc-link img');
 
-		release.tags = query.cnts('.dropdown ul a').slice(1);
+		release.tags = query.contents('.dropdown ul a').slice(1);
 		release.duration = query.duration('.dropdown p:first-child');
 
 		return release;
@@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) {
 async function scrapeScene({ query }, site, url) {
 	const release = { url, site };
 
-	release.title = query.cnt('#movie-header h2');
+	release.title = query.content('#movie-header h2');
-	release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
 
-	release.description = query.cnt('.container .mg-md');
+	release.description = query.content('.container .mg-md');
 	release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
 
-	release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
+	release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
-	release.tags = query.cnts('.tag-container a');
+	release.tags = query.contents('.tag-container a');
 
-	const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
+	const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
 	if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
 
 	release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
 	release.poster = query.poster();
 
-	const trailer = query.trailer();
+	const trailer = query.video('source');
 	if (trailer) release.trailer = { src: trailer };
 
 	if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
@@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) {
 
 async function fetchLatest(site, page = 1) {
 	const url = `${site.url}/movies/page-${page}`;
-	const res = await qu.getAll(url, '#content-main [class^="item"]');
+	const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
 
 	if (res.ok) {
-		return scrapeLatest(res.items, site);
+		return scrapeLatest(res.context, site);
 	}
 
 	return res.status;
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel, url);
+		return scrapeScene(res.context, channel, url);
 	}
 
 	return res.status;

src/scrapers/pinkyxxx.js

@@ -1,24 +1,42 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
-const http = require('../utils/http');
+
+function parseJson(res) {
+	// admin-ajax responds with JSON labeled as text/html
+	if (typeof res.data === 'object') {
+		return res.data;
+	}
+
+	try {
+		return JSON.parse(res.body);
+	} catch (_error) {
+		return null;
+	}
+}
 
 async function getTrailer(entryId, channel) {
-	const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
+	const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
 		action: 'vidplay',
 		vid: entryId,
 		type: 'promo',
 	}, {
-		encodeJSON: false,
+		form: true,
-		decodeJSON: true,
 	});
 
-	if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
+	const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
+
+	if (!videoLinks?.size1) {
 		return null;
 	}
 
-	const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
+	const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
-	const videoScriptRes = await http.get(scriptUrl);
+
+	if (!scriptUrl) {
+		return null;
+	}
+
+	const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
 
 	if (!videoScriptRes.ok) {
 		return null;
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
 	const videoData = videoDataString && JSON.parse(videoDataString);
 
 	const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
-	const posterUrlEl = videoScriptRes.document.createElement('span');
+	const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
-
-	posterUrlEl.innerHTML = posterUrlString;
 
 	return {
 		trailer: videoData?.url || null,
-		poster: posterUrlEl.textContent || null,
+		poster: poster || null,
 	};
 }
 
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
 		release.title = scene.title;
 		release.description = scene.description;
 
-		release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
+		release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
 
 		release.poster = scene.poster[0];
 		release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
 	});
 }
 
-async function scrapeScene({ query }, channel, baseScene) {
+async function scrapeScene({ query }, { entity, baseRelease }) {
 	const release = {};
 
-	release.title = query.cnt('.title');
+	release.title = query.content('.title');
-	release.description = query.cnt('.description');
+	release.description = query.content('.description');
 
-	release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
+	release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
 
-	const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
+	const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
 
-	release.poster = baseScene?.poster || poster;
+	release.poster = baseRelease?.poster || poster;
-	release.photos = poster && baseScene?.poster
+	release.photos = poster && baseRelease?.poster
-		? [poster, ...baseScene.photos]
+		? [poster, ...baseRelease.photos]
-		: baseScene?.photos;
+		: baseRelease?.photos;
 
 	release.trailer = trailer;
 
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
 }
 
 async function fetchLatest(channel, page) {
-	const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
+	const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
 		action: 'vls',
 		limit: 10,
 		offset: (page - 1) * 10,
 	}, {
-		encodeJSON: false,
+		form: true,
-		decodeJSON: true,
 	});
 
-	if (res.ok) {
+	const data = res.ok && parseJson(res);
-		return scrapeAll(res.body.listings);
+
+	if (data?.listings) {
+		return scrapeAll(data.listings);
 	}
 
-	return res.status;
+	return res.ok ? null : res.status;
-}
-
-async function fetchScene(url, channel, baseScene) {
-	const res = await qu.get(url);
-
-	if (res.ok) {
-		return scrapeScene(res.item, channel, baseScene);
-	}
-
-	return res.status;
 }
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
+	scrapeScene,
 };

src/scrapers/releases.js

@@ -39,7 +39,6 @@ const karups = require('./karups');
 const kellymadison = require('./kellymadison');
 const kink = require('./kink');
 const mariskax = require('./mariskax');
-// const analvids = require('./analvids');
 const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
@@ -74,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
 const testedefudelidade = require('./testedefudelidade');
 const theflourish = require('./theflourish');
 const tokyohot = require('./tokyohot');
-// const topwebmodels = require('./topwebmodels');
 const traxxx = require('./traxxx');
 const virtualtaboo = require('./virtualtaboo');
 const vixen = require('./vixen');

src/scrapers/rickysroom.js

@@ -1,22 +1,23 @@
 'use strict';
 
-const qu = require('../utils/q');
+const unprint = require('unprint');
+
 const { lbsToKg, feetInchesToCm } = require('../utils/convert');
 
 function scrapeScene(data, channel) {
 	const release = {};
 
 	release.entryId = data.id;
-	release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
+	release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
 
 	release.title = data.title;
 	release.description = data.description;
-	release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
+	release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
-	release.duration = qu.durationToSeconds(data.videos_duration);
+	release.duration = unprint.extractDuration(data.videos_duration);
 
 	release.actors = data?.models_thumbs.map((model) => ({
 		name: model.name,
-		url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
+		url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
 		avatar: model.thumb,
 	})) || data.models;
 
@@ -42,13 +43,12 @@ function scrapeScene(data, channel) {
 
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.contents?.data) {
+		if (data?.props?.pageProps?.contents?.data) {
 			const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
 
 			return scenes;
@@ -66,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
 		return baseRelease;
 	}
 
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.content) {
+		if (data?.props?.pageProps?.content) {
 			return scrapeScene(data.props.pageProps.content, channel);
 		}
 
@@ -86,11 +85,11 @@ function scrapeProfile(data, scenes, entity) {
 	const profile = {};
 
 	profile.entryId = data.id;
-	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
+	profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
 
 	profile.description = data.Bio || data.bio;
 	profile.birthPlace = data.Born || data.born;
-	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
+	profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
 
 	profile.measurements = data.Measurements || data.Measurements;
 	profile.height = feetInchesToCm(data.Height || data.height);
@@ -108,13 +107,12 @@ function scrapeProfile(data, scenes, entity) {
 
 async function fetchProfile({ slug }, entity) {
 	const url = `${entity.url}/models/${slug}`;
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.model) {
+		if (data?.props?.pageProps?.model) {
 			return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
 		}
 

src/scrapers/teamskeet.js

@@ -2,7 +2,6 @@
 
 const unprint = require('unprint');
 
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -34,7 +33,8 @@ async function fetchTrailerUrl(videoId, entity) {
 		const token = res.data.token;
 
 		if (token) {
-			return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`;
+			// ffmpeg fails to read the DASH manifest over a single connection, HLS works
+			return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.m3u8?parentOrigin=${entity.origin}`;
 		}
 	}
 
@@ -47,17 +47,18 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
 	// release.entryId = scene.id; // legacy
 	release.entryId = scene.itemId;
 
-	release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
+	release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
 
 	release.title = scene.title;
 	release.description = scene.description;
 	release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
+	release.duration = scene.d;
 
 	release.actors = scene.models?.map((model) => ({
 		name: model.modelName || model.name || model.title,
 		avatar: model.img,
 		gender: model.gender,
-		url: `${channel.url}/models/${model.modelId || model.id}`,
+		url: `${channel.origin}/models/${model.modelId || model.id}`,
 	}));
 
 	if (scene.img) {
@@ -93,16 +94,16 @@ function scrapeAll(scenes, channel, parameters) {
 
 async function fetchLatest(channel, page = 1, { parameters }) {
 	// url: 'https://www.pervz.com/series/pervz-features',
-	const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
+	const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
 
 	const url = seriesId
 		? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
 		: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
 
-	const res = await http.get(url);
+	const res = await unprint.get(url);
 
-	if (res.ok) {
+	if (res.ok && res.data?.hits) {
-		return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
+		return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
 	}
 
 	return res.status;
@@ -114,27 +115,22 @@ async function fetchScene(url, channel, baseScene, { parameters, includeTrailers
 		return baseScene;
 	}
 
-	const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
+	const sceneSlug = new URL(url).pathname.match(/\/([\w-]+)\/?$/)?.[1];
 
-	const res = await unprint.get(url, {
+	if (!sceneSlug) {
-		parser: {
+		return null;
-			runScripts: 'dangerously',
+	}
-		},
-	});
 
-	if (res.ok) {
+	// scene page no longer embeds state, look up the scene in the elastic API by its slug instead
-		const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent;
+	// legacy URLs use the numeric item ID, which 404s on the site but can still be looked up
+	const isItemId = /^\d+$/.test(sceneSlug);
+	const res = await unprint.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(${isItemId ? `itemId:${sceneSlug}` : `id:"${sceneSlug}"`} AND type:video)&size=1`);
 
-		res.context.window.fetch = () => {}; // suppress fetch missing error
+	if (res.ok && res.data?.hits) {
+		const scene = res.data.hits.hits.find(({ _source: hit }) => (isItemId ? String(hit.itemId) : hit.id) === sceneSlug)?._source;
 
-		if (!videos) {
+		if (scene) {
-			return null;
+			return scrapeScene(scene, channel, parameters, includeTrailers);
-		}
-
-		const video = videos?.[sceneSlug] || Object.values(videos)[0];
-
-		if (video) {
-			return scrapeScene(video, channel, parameters, includeTrailers);
 		}
 
 		return null;

src/scrapers/topwebmodels.js

@@ -1,144 +0,0 @@
-'use strict';
-
-const mime = require('mime');
-
-const qu = require('../utils/qu');
-const http = require('../utils/http');
-const slugify = require('../utils/slugify');
-const { convert } = require('../utils/convert');
-
-function scrapeSceneX(scene) {
-	const release = {};
-
-	release.entryId = scene.id;
-	release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
-
-	release.title = scene.title;
-	release.description = scene.description;
-
-	release.duration = qu.durationToSeconds(scene.videos_duration);
-	release.date = new Date(scene.release_date);
-
-	release.actors = scene.models
-		.map((actor) => (/&/.test(actor.name)
-			? actor.name.split(/\s*&\s*/)
-			: {
-				name: actor.name,
-				gender: actor.gender || null,
-				avatar: actor.thumb,
-				url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
-			}))
-		.flat();
-
-	release.stars = scene.rating;
-	release.tags = scene.tags.map((tag) => tag.name);
-
-	if (mime.getType(scene.thumb) === 'image/gif') {
-		release.teaser = scene.thumb;
-	} else {
-		release.poster = scene.thumb;
-	}
-
-	release.channel = slugify(scene.sites[0]?.name, '');
-
-	return release;
-}
-
-function scrapeAll(scenes) {
-	return scenes.map(scrapeSceneX);
-}
-
-async function scrapeProfile(actor, options) {
-	const profile = {};
-
-	profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
-	profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
-	profile.gender = actor.attributes.gender || null;
-
-	profile.height = convert(actor.attributes.height.value, 'cm');
-	profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
-
-	const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
-
-	profile.bust = Number(bust);
-	profile.cup = cup;
-	profile.waist = Number(waist);
-	profile.hip = Number(hip);
-
-	profile.ethnicity = actor.attributes.ethnicity.value;
-	profile.birthPlace = actor.attributes.born.value;
-
-	profile.eyes = actor.attributes.eyes.value;
-	profile.hairColor = actor.attributes.hair.value.split('/')[0];
-
-	profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
-	profile.avatar = actor.thumb;
-
-	if (options.includeActorScenes) {
-		const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
-
-		if (res.ok) {
-			profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
-		}
-	}
-
-	return profile;
-}
-
-async function fetchLatest(channel, page) {
-	const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
-		headers: {
-			Referer: 'https://tour.topwebmodels.com',
-			'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
-			'x-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (res.ok) {
-		return scrapeAll(res.body.videos.items);
-	}
-
-	return res.status;
-}
-
-async function fetchScene(url) {
-	const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
-
-	if (res.ok) {
-		return {
-			...scrapeSceneX(res.window.__DATA__.data.video),
-			...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
-			poster: res.window.__DATA__.data.file_poster,
-		};
-	}
-
-	return res.status;
-}
-
-async function fetchProfile(baseActor, entity, options) {
-	const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
-		headers: {
-			Referer: 'https://tour.topwebmodels.com',
-			'api-key': entity.parameters?.apiKey,
-			'x-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (!searchRes.ok) {
-		return searchRes.status;
-	}
-
-	const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
-
-	if (actor) {
-		return scrapeProfile(actor, options);
-	}
-
-	return null;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchScene,
-	fetchProfile,
-};

src/store-releases.js

@@ -12,7 +12,6 @@ const slugify = require('./utils/slugify');
 const bulkInsert = require('./utils/bulk-insert');
 const chunk = require('./utils/chunk');
 const redis = require('./redis');
-const { formatDate } = require('./utils/qu');
 const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
 const { curateEntity } = require('./entities');
@@ -32,7 +31,7 @@ const commonContext = {
 async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
 	const slugBase = release.title
 		|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
-		|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
+		|| (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`)
 		|| null;
 
 	const slug = slugify(slugBase, '-', {

src/tags.js

@@ -79,13 +79,14 @@ function withRelations(queryBuilder, withMedia) {
 }
 
 async function matchTags(rawTags) {
-	const tags = rawTags
+	const tags = Array.from(new Set(rawTags
-		.map((tag) => tag?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
+		.map((tag) => (typeof tag === 'string' ? tag : null)?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
-		.filter(Boolean);
+		.filter(Boolean)));
 
+	// array binding, large batches can easily exceed the bindings limit otherwise
 	const tagEntries = await knex('tags')
 		.select('tags.id', 'tags.name', 'tags.alias_for', 'tags.implied_tag_ids')
-		.whereIn(knex.raw('lower(tags.name)'), tags);
+		.whereRaw('lower(tags.name) = ANY(:tags)', { tags });
 
 	const tagIdsBySlug = tagEntries
 		.reduce((acc, tag) => ({

src/tools/dogfart.js

@@ -1,76 +0,0 @@
-'use strict';
-
-const fs = require('fs').promises;
-
-const Promise = require('bluebird');
-
-const qu = require('../utils/qu');
-
-const qualities = {
-	sm: 360,
-	med: 480,
-	big: 720,
-};
-
-/*
-async function scrape() {
-	const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
-		const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
-
-		if (res.ok) {
-			return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
-		}
-
-		return [];
-	}, { concurrency: 1 });
-
-	const urls = urlsByPage.flat();
-
-	await fs.writeFile('./dogfart-links', urls.join('\n'));
-
-	console.log(`Saved ${urls.length} URLs to file`);
-}
-
-async function compare() {
-	const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
-	const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
-
-	const newLinks = newLinksFile.split('\n').filter(Boolean);
-	const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
-
-	const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
-
-	await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
-
-	console.log(getLinks);
-}
-*/
-
-async function scrapeMembers() {
-	const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
-		const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
-
-		if (res.ok) {
-			return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
-				url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
-				actors: query.contents('a[href*="model.php"]'),
-				trailer: query.urls('.trailer-link a').map((url) => ({
-					src: url,
-					quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
-				})),
-			}));
-		}
-
-		return [];
-	}, { concurrency: 1 });
-
-	const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
-
-	console.log(urls);
-
-	await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
-
-	console.log(`Saved ${urls.length} URLs to file`);
-}
-
-scrapeMembers();

src/utils/media.js

@@ -1,110 +0,0 @@
-'use strict';
-
-const config = require('config');
-const fs = require('fs');
-const fsPromises = require('fs').promises;
-const Promise = require('bluebird');
-const blake2 = require('blake2');
-const sharp = require('sharp');
-const { nanoid } = require('nanoid');
-const { PassThrough } = require('stream');
-
-const http = require('./http');
-
-function getMemoryUsage() {
-	return process.memoryUsage().rss / (10 ** 6);
-}
-
-let peakMemoryUsage = getMemoryUsage();
-
-async function fetchSource(link) {
-	const id = nanoid();
-
-	const hasher = new blake2.Hash('blake2b');
-	hasher.setEncoding('hex');
-
-	const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
-	const tempFileStream = fs.createWriteStream(tempFilePath);
-	const hashStream = new PassThrough();
-
-	hashStream.on('data', (chunk) => hasher.write(chunk));
-
-	try {
-		const res = await http.get(link, null, {
-			stream: true,
-			transforms: [hashStream],
-			destination: tempFileStream,
-			timeout: 5000,
-		});
-
-		if (!res.ok) {
-			throw new Error(res.status);
-		}
-
-		hasher.end();
-		const hash = hasher.read();
-
-		const memoryUsage = getMemoryUsage();
-		peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
-
-		console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
-
-		return {
-			id,
-			path: tempFilePath,
-			hash,
-		};
-	} catch (error) {
-		await fsPromises.unlink(tempFilePath);
-
-		throw error;
-	}
-}
-
-async function init() {
-	const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
-	const links = linksFile.split('\n').filter(Boolean);
-
-	await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
-
-	console.time('thumbs');
-
-	const files = await Promise.map(links, async (link) => {
-		try {
-			return await fetchSource(link);
-		} catch (error) {
-			console.log(`Failed to fetch ${link}: ${error.message}`);
-			return null;
-		}
-	});
-
-	await Promise.map(files.filter(Boolean), async (file) => {
-		const image = sharp(file.path).jpeg();
-
-		const [{ width, height }, { size }] = await Promise.all([
-			image.metadata(),
-			fsPromises.stat(file.path),
-		]);
-
-		await Promise.all([
-			image
-				.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
-			image
-				.resize({
-					height: config.media.thumbnailSize,
-					withoutEnlargement: true,
-				})
-				.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
-		]);
-
-		const memoryUsage = getMemoryUsage();
-		peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
-
-		console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
-	}, { concurrency: 10 });
-
-	console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
-	console.timeEnd('thumbs');
-}
-
-init();

src/utils/q.js

@@ -1,5 +0,0 @@
-'use strict';
-
-const qu = require('./qu');
-
-module.exports = qu;

src/utils/qu.js

@@ -1,682 +0,0 @@
-'use strict';
-
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-const http = require('./http');
-const virtualConsole = require('./virtual-console')(__filename);
-
-const { window: globalWindow } = new JSDOM('', { virtualConsole });
-
-function trim(str) {
-	if (typeof str !== 'string') {
-		return str;
-	}
-
-	return str.trim().replace(/\s+/g, ' ');
-}
-
-function extractDate(dateString, format, match) {
-	if (!dateString) {
-		return null;
-	}
-
-	if (match) {
-		const dateStamp = trim(dateString).match(match);
-
-		if (dateStamp) {
-			const dateValue = moment.utc(dateStamp[0], format);
-
-			return dateValue.isValid() ? dateValue.toDate() : null;
-		}
-		return null;
-	}
-
-	const dateValue = moment.utc(trim(dateString), format);
-
-	return dateValue.isValid() ? dateValue.toDate() : null;
-}
-
-function formatDate(dateValue, format, inputFormat) {
-	if (inputFormat) {
-		return moment(dateValue, inputFormat).format(format);
-	}
-
-	return moment(dateValue).format(format);
-}
-
-function durationToSeconds(durationString, match) {
-	const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
-
-	if (durationMatch) {
-		const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3);
-
-		return moment.duration(segments.join(':')).asSeconds();
-	}
-
-	return null;
-}
-
-function prefixUrl(urlValue, origin, protocol = 'https') {
-	if (!urlValue) {
-		return null;
-	}
-
-	if (/^http/.test(urlValue)) {
-		return urlValue;
-	}
-
-	if (protocol && /^\/\//.test(urlValue)) {
-		return `${protocol}:${urlValue}`;
-	}
-
-	if (origin && /^\//.test(urlValue)) {
-		return `${origin}${urlValue}`;
-	}
-
-	if (origin && /^\.\//.test(urlValue)) {
-		return `${origin}${urlValue.slice(1)}`;
-	}
-
-	if (origin) {
-		return `${origin}/${urlValue}`;
-	}
-
-	return urlValue;
-}
-
-function iterateXPathResult(iterator, results = []) {
-	const element = iterator.iterateNext();
-
-	if (element) {
-		return iterateXPathResult(iterator, [...results, element]);
-	}
-
-	return results;
-}
-
-function getElements(context, selector, first = false) {
-	if (!selector) {
-		return context;
-	}
-
-	if (/^\/\//.test(selector)) {
-		// XPath selector
-		const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
-
-		if (first) {
-			return iterator.iterateNext();
-		}
-
-		return iterateXPathResult(iterator);
-	}
-
-	if (first) {
-		return context.querySelector(selector);
-	}
-
-	return Array.from(context.querySelectorAll(selector));
-}
-
-function q(context, selectors, attrArg, applyTrim = true) {
-	if (!selectors && context.nodeName === '#document') {
-		return null;
-	}
-
-	const attr = attrArg === true ? 'textContent' : attrArg;
-	const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
-
-	if (!element) {
-		return null;
-	}
-
-	if (attr) {
-		const value = element[attr] || element.getAttribute(attr);
-
-		return applyTrim && typeof value === 'string' ? trim(value) : value;
-	}
-
-	return element;
-}
-
-function all(context, selectors, attrArg, applyTrim = true) {
-	const attr = attrArg === true ? 'textContent' : attrArg;
-	const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null);
-
-	if (!Array.isArray(elements)) {
-		return [];
-	}
-
-	if (attr) {
-		return elements.map((el) => q(el, null, attr, applyTrim));
-	}
-
-	return elements;
-}
-
-function exists(context, selector) {
-	return !!q(context, selector);
-}
-
-function count(context, selector) {
-	return all(context, selector)?.length || 0;
-}
-
-function content(context, selector, applyTrim = true) {
-	return q(context, selector, 'textContent', applyTrim);
-}
-
-function contents(context, selector, applyTrim) {
-	return all(context, selector, 'textContent', applyTrim);
-}
-
-function html(context, selector) {
-	const el = q(context, selector, null, true);
-
-	return el && el.innerHTML;
-}
-
-function htmls(context, selector) {
-	const els = all(context, selector, null, true);
-
-	return els.map((el) => el.innerHTML);
-}
-
-function execute(context, selector = 'script') {
-	const scripts = htmls(context, selector);
-	const originalGlobal = Object.fromEntries(Object.entries(global));
-
-	const errors = scripts?.reduce((accErrors, script) => {
-		try {
-			Function(script)(); /* eslint-disable-line no-new-func */
-
-			return accErrors;
-		} catch (error) {
-			// the script failed
-			return [...accErrors, error];
-		}
-	}, []);
-
-	const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => {
-		if (originalGlobal[key] !== value) {
-			delete global[key];
-			return true;
-		}
-
-		return false;
-	}));
-
-	return {
-		...data,
-		errors,
-	};
-}
-
-function json(context, selector) {
-	const el = q(context, selector, null, true);
-
-	try {
-		return JSON.parse(el?.innerHTML);
-	} catch (error) {
-		return null;
-	}
-}
-
-function jsons(context, selector) {
-	const els = all(context, selector, null, true);
-
-	return els.map((el) => {
-		try {
-			return JSON.parse(el?.innerHTML);
-		} catch (error) {
-			return null;
-		}
-	});
-}
-
-function texts(context, selector, applyTrim = true, filter = true) {
-	const el = q(context, selector, null, applyTrim);
-	if (!el) return null;
-
-	const nodes = Array.from(el.childNodes)
-		.filter((node) => node.nodeName === '#text')
-		.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
-
-	return filter ? nodes.filter(Boolean) : nodes;
-}
-
-function text(context, selector, applyTrim = true) {
-	const nodes = texts(context, selector, applyTrim, true);
-	if (!nodes) return null;
-
-	const textValue = nodes.join(' ');
-
-	return applyTrim ? trim(textValue) : textValue;
-}
-
-function removeStyleFunctionSpaces(el) {
-	// jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png )
-	el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`));
-}
-
-function style(context, selector, styleAttr) {
-	const el = q(context, selector);
-
-	if (el?.hasAttribute('style')) {
-		removeStyleFunctionSpaces(el);
-
-		return styleAttr ? el.style[styleAttr] : el.style;
-	}
-
-	return null;
-}
-
-function styles(context, selector, styleAttr) {
-	const elStyles = Array.from(context.querySelectorAll(selector), (el) => {
-		removeStyleFunctionSpaces(el);
-
-		return styleAttr ? el.style[styleAttr] : el.style;
-	});
-
-	return elStyles;
-}
-
-function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
-	const value = q(context, selector, attr);
-
-	if (value && match) {
-		return Number(value.match(match)?.[matchIndex]);
-	}
-
-	if (value) {
-		return Number(value);
-	}
-
-	return null;
-}
-
-function meta(context, selector, attrArg = 'content', applyTrim = true) {
-	if (/meta\[.*\]/.test(selector)) {
-		return q(context, selector, attrArg, applyTrim);
-	}
-
-	return q(context, `meta[${selector}]`, attrArg, applyTrim);
-}
-
-function date(context, selector, format, match, attr = 'textContent') {
-	const dateString = q(context, selector, attr, true);
-
-	if (!dateString) return null;
-
-	return extractDate(dateString, format, match);
-}
-
-function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') {
-	const timeString = q(context, selector, attr, 'textContent');
-
-	if (!timeString) {
-		return null;
-	}
-
-	const timeMatch = timeString.match(match);
-
-	if (timeMatch) {
-		const [n, period] = timeMatch.slice(1);
-		const thenDate = moment.utc().subtract(Number(n), period);
-
-		return {
-			date: thenDate.toDate(),
-			precision: period.replace(/s$/, ''),
-		};
-	}
-
-	return null;
-}
-
-function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
-	const imageEl = (attr && q(context, selector, attr))
-		|| q(context, selector, 'data-src')
-		|| q(context, selector, 'src');
-
-	return prefixUrl(imageEl, origin, protocol);
-}
-
-function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
-	const attribute = attr
-		|| (q(context, selector, 'data-src') && 'data-src')
-		|| (q(context, selector, 'src') && 'src');
-
-	const imageEls = all(context, selector, attribute);
-
-	return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
-}
-
-function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
-	const urlEl = q(context, selector, attr);
-	const prefixedUrl = prefixUrl(urlEl, origin, protocol);
-
-	if (prefixedUrl && object) {
-		return new URL(prefixedUrl);
-	}
-
-	return prefixedUrl;
-}
-
-function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
-	const urlEls = all(context, selector, attr);
-
-	return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
-}
-
-function sourceSet(context, selector, attr = 'srcset', options = {}) {
-	const srcset = q(context, selector, attr);
-
-	if (!srcset) {
-		return null;
-	}
-
-	const sources = srcset
-		.split(/\s*,\s*/)
-		.map((source) => {
-			const [link, descriptor] = source.split(' ');
-
-			if (link) {
-				return {
-					descriptor: descriptor || 'fallback',
-					url: prefixUrl(link, options.origin, options.protocol),
-				};
-			}
-
-			return null;
-		})
-		.filter(Boolean)
-		.sort((sourceA, sourceB) => {
-			if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) {
-				return -1;
-			}
-
-			if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) {
-				return 1;
-			}
-
-			return 0;
-		});
-
-	if (options.includeDescriptor) {
-		return sources;
-	}
-
-	return sources.map((source) => source.url);
-}
-
-function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
-	const posterEl = q(context, selector, attr);
-
-	return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
-}
-
-function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
-	const trailerEl = q(context, selector, attr);
-
-	return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
-}
-
-function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
-	const trailerEls = all(context, selector, attr);
-
-	return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
-}
-
-function duration(context, selector, match, attr = 'textContent') {
-	const durationString = q(context, selector, attr);
-
-	if (!durationString) {
-		return null;
-	}
-
-	const durationMatch = durationToSeconds(durationString, match);
-
-	if (durationMatch) {
-		return durationMatch;
-	}
-
-	const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i);
-
-	if (timestampMatch) {
-		const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0;
-		const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0;
-		const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0;
-
-		return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds);
-	}
-
-	return null;
-}
-
-const legacyFuncs = {
-	q,
-	qa: all,
-	qall: all,
-	qd: date,
-	qdate: date,
-	qh: html,
-	qhtml: html,
-	qi: image,
-	qimage: image,
-	qimages: images,
-	qis: images,
-	ql: duration,
-	qlength: duration,
-	qm: meta,
-	qmeta: meta,
-	qp: poster,
-	qposter: poster,
-	qs: all,
-	qt: video,
-	qtext: text,
-	qtexts: texts,
-	qtrailer: video,
-	qtrailers: videos,
-	qts: videos,
-	qtx: text,
-	qtxs: texts,
-	qtxt: text,
-	qtxts: texts,
-	// qu: url,
-	qurl: url,
-	qurls: urls,
-	qus: urls,
-};
-
-const quFuncs = {
-	all,
-	cnt: content,
-	cnts: contents,
-	content,
-	contents,
-	count,
-	date,
-	dateAgo,
-	dur: duration,
-	duration,
-	el: q,
-	element: q,
-	execute,
-	exec: execute,
-	exists,
-	html,
-	htmls,
-	image,
-	images,
-	img: image,
-	imgs: images,
-	json,
-	jsons,
-	length: duration,
-	meta,
-	num: number,
-	number,
-	poster,
-	q,
-	sourceSet,
-	sources: sourceSet,
-	srcs: sourceSet,
-	srcset: sourceSet,
-	style,
-	styles,
-	text,
-	texts,
-	trailer: video,
-	url,
-	urls,
-	video,
-	videos,
-};
-
-function init(context, selector, window) {
-	if (!context) {
-		return null;
-	}
-
-	const element = selector ? context.querySelector(selector) : context;
-
-	if (!element) {
-		return null;
-	}
-
-	const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
-		.reduce((acc, [key, func]) => ({
-			...acc,
-			[key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context
-				? func(...args)
-				: func(element, ...args)),
-		}), {});
-
-	const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
-		.reduce((acc, [key, func]) => ({
-			...acc,
-			[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
-				? func(element, ...args)
-				: func(...args)),
-		}), {});
-
-	return {
-		element,
-		el: element,
-		html: element.outerHTML || element.body.outerHTML,
-		text: trim(element.textContent),
-		...(window && {
-			window,
-			document: window.document,
-		}),
-		...legacyContextFuncs,
-		qu: quContextFuncs,
-		query: quContextFuncs,
-	};
-}
-
-function initAll(context, selector, window) {
-	if (Array.isArray(context)) {
-		return context.map((element) => init(element, null, window));
-	}
-
-	return Array.from(context.querySelectorAll(selector))
-		.map((element) => init(element, null, window));
-}
-
-function extract(htmlValue, selector, options) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
-
-	return init(window.document, selector, window);
-}
-
-function extractAll(htmlValue, selector, options) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
-
-	return initAll(window.document, selector, window);
-}
-
-async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
-	const res = await (method === 'post'
-		? http.post(urlValue, body, { ...options, headers })
-		: http[method](urlValue, {
-			...options,
-			headers,
-			parse: true,
-		}));
-
-	if (res.ok) {
-		const item = queryAll
-			? initAll(res.document, selector, res.window)
-			: init(res.document, selector, res.window);
-
-		return {
-			item,
-			items: all ? item : [item],
-			res,
-			ok: true,
-			status: res.statusCode,
-			...res,
-		};
-	}
-
-	return {
-		item: null,
-		items: [],
-		res,
-		ok: false,
-		status: res.statusCode,
-		...res,
-	};
-}
-
-async function get(urlValue, selector, headers, options) {
-	return request('get', urlValue, null, selector, headers, options, false);
-}
-
-async function post(urlValue, body, selector, headers, options) {
-	return request('post', urlValue, body, selector, headers, options, false);
-}
-
-async function getAll(urlValue, selector, headers, options) {
-	return request('get', urlValue, null, selector, headers, options, true);
-}
-
-async function postAll(urlValue, body, selector, headers, options) {
-	return request('post', urlValue, body, selector, headers, options, true);
-}
-
-function session(headers, options) {
-	return http.session(headers, options);
-}
-
-module.exports = {
-	extractDate,
-	extract,
-	extractAll,
-	durationToSeconds,
-	init,
-	initAll,
-	formatDate,
-	get,
-	getAll,
-	http,
-	fetch: get,
-	fetchAll: getAll,
-	context: init,
-	contextAll: initAll,
-	ed: extractDate,
-	ex: extract,
-	exa: extractAll,
-	fd: formatDate,
-	parseDate: extractDate,
-	ctx: init,
-	ctxa: initAll,
-	geta: getAll,
-	qu: quFuncs,
-	query: quFuncs,
-	post,
-	postAll,
-	prefixUrl,
-	session,
-	...legacyFuncs,
-};

src/utils/timeout.js

@@ -1,20 +0,0 @@
-'use strict';
-
-const http = require('./http');
-
-const sleep = 5000;
-const timeout = 1000;
-
-async function init() {
-	try {
-		const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
-			timeout,
-		});
-
-		console.log(res.statusCode);
-	} catch (error) {
-		console.log(error);
-	}
-}
-
-init();