1.257.6

.eslintrc

@@ -5,11 +5,12 @@
 		"node": true
 	},
     "parserOptions": {
-        "parser": "@babel/eslint-parser",
         "ecmaVersion": 2024,
         "sourceType": "script"
     },
     "rules": {
+		"import/no-unresolved": ["error", {"ignore": ["^yargs/helpers$", "^object-merge-advanced$"]}],
+		"import/extensions": ["error", "ignorePackages", {"pattern": {"js": "never", "mjs": "never", "jsx": "never"}, "pathGroupOverrides": [{"pattern": "@noble/hashes/**", "action": "ignore"}]}],
         "indent": ["error", "tab"],
         "no-tabs": "off",
 		"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],

.nvmrc

@@ -1 +1 @@
-21.2.0
+26.13.0

README.md

@@ -2,9 +2,9 @@
 **NSFW** - The latest releases from your favorite porn studios in one place.
 
 ## Install & run
-Use [nvm](https://github.com/creationix/nvm) to install NodeJS v16.8.0 or newer. Download and unpack or clone this repository, and run the following in the root directory:
+Use [nvm](https://github.com/creationix/nvm) to install NodeJS v26.13.0, and make sure it's active (`nvm use`). Download and unpack or clone this repository, and run the following in the root directory:
 
-`npm install`
+`npm ci`
 
 ### Set up database
 Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user.
@@ -18,6 +18,11 @@ Do not modify `config/default.js`, but instead create a copy at `config/local.js
 
 You can also use `npm run flush` to run both steps at once, and wipe the database completely later.
 
+### Install browser
+Unless configured to use unprint's remote browser server, a browser needs to be installed locally:
+
+`npx patchright install chromium`
+
 #### Networks and channels
 To scrape the networks and channels available in the database, you can configure `include` and `exclude` lists. To include all available channels and only use the `exclude` list, leave the `include` parameter unconfigured. The `exclude` lists will exclude channels and child networks from networks on the `include` lists, but not vice versa. That is, if the `include` list includes a network and the `exclude` list excludes one of that network's channels, the channel will not be scraped. However, if the `include` list includes a channel, and the `exclude` list includes its parent network, the channel will be scraped.
 
@@ -50,11 +55,7 @@ exclude: {
 },
 ```
 
-### Building
+### Thumbnails
-To build traxxx, run the following command:
-
-`npm run build`
-
 To generate thumbnails for new logos and tag photos, install ImageMagick and run:
 
 `npm run logos-thumbs`

assets/20240125011700_manticore.js

@@ -1,138 +0,0 @@
-const config = require('config');
-const manticore = require('manticoresearch');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-
-const scenesFields = `
-	 id int,
-	 title text,
-	 title_filtered text,
-	 shoot_id text,
-	 channel_id int,
-	 channel_name text,
-	 channel_slug text,
-	 network_id int,
-	 network_name text,
-	 network_slug text,
-	 studio_id int,
-	 studio_name text,
-	 studio_slug text,
-	 entity_ids multi,
-	 actor_ids multi,
-	 actors text,
-	 tag_ids multi,
-	 tags text,
-	 movie_ids multi,
-	 movies text,
-	 serie_ids multi,
-	 series text,
-	 meta text,
-	 date timestamp,
-	 is_showcased bool,
-	 created_at timestamp,
-	 effective_date timestamp,
-	 stashed int,
-	 dupe_index int
-`;
-
-const moviesFields = `
-	 id int,
-	 title text,
-	 title_filtered text,
-	 channel_id int,
-	 channel_name text,
-	 channel_slug text,
-	 network_id int,
-	 network_name text,
-	 network_slug text,
-	 entity_ids multi,
-	 actor_ids multi,
-	 actors text,
-	 tag_ids multi,
-	 tags text,
-	 meta text,
-	 date timestamp,
-	 has_cover bool,
-	 created_at timestamp,
-	 effective_date timestamp,
-	 stashed int,
-	 stashed_scenes int,
-	 stashed_total int,
-	 dupe_index int
-`;
-
-const actorsFields = `
-	 id int,
-	 name text,
-	 slug string,
-	 gender string,
-	 date_of_birth timestamp,
-	 country string,
-	 has_avatar bool,
-	 mass int,
-	 height int,
-	 cup string,
-	 natural_boobs int,
-	 penis_length int,
-	 penis_girth int,
-	 stashed int,
-	 scenes int
-`;
-
-exports.up = async () => {
-	try {
-		await utilsApi.sql(`create table if not exists scenes (${scenesFields})`);
-
-		await utilsApi.sql(`create table if not exists scenes_stashed (
-			scene_id int,
-			stash_id int,
-			user_id int,
-			created_at timestamp
-		)`);
-
-		await utilsApi.sql(`create table if not exists movies (${moviesFields})`);
-
-		await utilsApi.sql(`create table if not exists movies_stashed (
-			movie_id int,
-			stash_id int,
-			user_id int,
-			created_at timestamp
-		)`);
-
-		await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`);
-
-		await utilsApi.sql(`create table if not exists actors_stashed (
-			actor_id int,
-			stash_id int,
-			user_id int,
-			created_at timestamp
-		)`);
-
-		/*
-		await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id'));
-		await knex.schema.alterTable('stashes_movies', (table) => table.increments('id'));
-		await knex.schema.alterTable('stashes_actors', (table) => table.increments('id'));
-		await knex.schema.alterTable('stashes_series', (table) => table.increments('id'));
-		*/
-	} catch (error) {
-		console.log(error);
-	}
-};
-
-exports.down = async (knex) => {
-	await utilsApi.sql('drop table if exists scenes');
-	await utilsApi.sql('drop table if exists scenes_stashed');
-	await utilsApi.sql('drop table if exists movies');
-	await utilsApi.sql('drop table if exists movies_stashed');
-	await utilsApi.sql('drop table if exists actors');
-	await utilsApi.sql('drop table if exists actors_stashed');
-
-	await knex.schema.alterTable('stashes_scenes', (table) => table.dropColumn('id'));
-	await knex.schema.alterTable('stashes_movies', (table) => table.dropColumn('id'));
-	await knex.schema.alterTable('stashes_actors', (table) => table.dropColumn('id'));
-	await knex.schema.alterTable('stashes_series', (table) => table.dropColumn('id'));
-};

config/default.js

@@ -181,6 +181,8 @@ module.exports = {
 			'traxxx',
 			// porn doe
 			'forbondage',
+			// prevent huge influx
+			'manyvids',
 		],
 	},
 	profiles: null,
@@ -214,12 +216,7 @@ module.exports = {
 	},
 	bypass: {
 		browser: {
-			enable: false,
 			clientRetirement: 20,
-			hostnames: [ // these can run in the same browser session
-				'www.kink.com',
-				'store2.psmcdn.net', // Team Skeet API
-			],
 			remote: {
 				enable: false,
 				use: false,
@@ -227,15 +224,6 @@ module.exports = {
 				key: null,
 			},
 		},
-		cloudflare: {
-			enable: false,
-			auto: true, // try bypass when CF challenge is detected
-			path: 'http://localhost:8191/v1',
-			sharedHostnames: [ // these can run in the same browser session
-				'store2.psmcdn.net', // Team Skeet API
-			],
-			independentHostnames: [], // these must run in their own browser session
-		},
 	},
 	limits: {
 		default: {

ecosystem.config.js

@@ -1,28 +0,0 @@
-const config = require('config');
-
-module.exports = {
-	apps: [
-		{
-			name: 'traxxx',
-			script: 'src/init.js',
-			exec_mode: 'cluster',
-			instances: 2,
-			restart_delay: 3000,
-			args: '--server',
-			merge_logs: true,
-			env: {
-				NODE_ENV: 'production',
-			},
-		},
-		...(config.analytics.enabled ? [{
-			name: 'umami',
-			script: 'npm start',
-			cwd: process.env.UMAMI_DIR || '../umami',
-			restart_delay: 3000,
-			merge_logs: true,
-			env: {
-				NODE_ENV: 'production',
-			},
-		}] : []),
-	],
-};

migrations/20190325001339_releases.js

@@ -40,6 +40,7 @@ exports.up = async (knex) => {
 			.index();
 
 		table.text('name');
+		table.string('name_stylized');
 		table.text('slug', 32);
 
 		table.text('type')
@@ -126,10 +127,15 @@ exports.up = async (knex) => {
 			.defaultTo(knex.fn.now());
 	});
 
+	await knex.schema.createMaterializedView('media_sfw', (view) => {
+		view.as(knex('media').select('id').where('is_sfw', true));
+	});
+
+	await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
+
 	await knex.raw(`
 		CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media
+			SELECT id FROM media_sfw
-			WHERE is_sfw = true
 			ORDER BY random()
 			LIMIT 1;
 		$$ LANGUAGE sql STABLE;
@@ -143,6 +149,36 @@ exports.up = async (knex) => {
 			.defaultTo(knex.raw('get_random_sfw_media_id()'));
 	});
 
+	await knex.schema.createTable('media_biometrics', (table) => {
+		table.increments('id');
+
+		table.string('media_id', 21)
+			.references('id')
+			.inTable('media')
+			.notNullable()
+			.onDelete('cascade');
+
+		table.integer('width');
+		table.integer('height');
+
+		table.integer('face_index')
+			.notNullable()
+			.defaultTo(0);
+
+		table.json('biometrics');
+		table.specificType('embedding', 'vector(1024)');
+
+		table.datetime('updated_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+
+		table.unique(['media_id', 'face_index']);
+	});
+
 	await knex.schema.createTable('tags_groups', (table) => {
 		table.increments('id');
 
@@ -335,7 +371,7 @@ exports.up = async (knex) => {
 
 		table.decimal('shoe_size');
 		table.integer('leg');
-		table.integer('foot');
+		table.decimal('foot');
 		table.integer('thigh');
 
 		table.string('blood_type');
@@ -357,6 +393,9 @@ exports.up = async (knex) => {
 
 		table.string('agency');
 
+		table.boolean('allow_global_match');
+		table.text('comment');
+
 		table.text('avatar_media_id', 21)
 			.references('id')
 			.inTable('media');
@@ -443,7 +482,7 @@ exports.up = async (knex) => {
 
 		table.decimal('shoe_size');
 		table.integer('leg');
-		table.integer('foot');
+		table.decimal('foot');
 		table.integer('thigh');
 
 		table.string('blood_type');
@@ -655,6 +694,7 @@ exports.up = async (knex) => {
 			.inTable('actors');
 
 		table.integer('profile_id', 12)
+			.notNullable()
 			.references('id')
 			.inTable('actors_profiles')
 			.onDelete('cascade');
@@ -672,8 +712,6 @@ exports.up = async (knex) => {
 		table.unique(['profile_id', 'media_id']);
 	});
 
-	await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);');
-
 	await knex.schema.createTable('actors_photos', (table) => {
 		table.integer('actor_id', 12)
 			.notNullable()
@@ -713,12 +751,20 @@ exports.up = async (knex) => {
 		table.datetime('pinged_at');
 		table.datetime('verified_at');
 
-		table.unique(['actor_id', 'platform', 'handle']);
 		table.unique(['actor_id', 'url']);
 	});
 
 	await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);');
 	await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);');
+	await knex.raw('CREATE UNIQUE INDEX actors_socials_actor_id_platform_handle_lower_unique ON actors_socials (actor_id, platform, lower(handle));');
+
+	await knex.schema.createTable('languages', (table) => {
+		table.string('alpha2')
+			.primary();
+
+		table.text('name');
+		table.text('name_native');
+	});
 
 	await knex.schema.createTable('releases', (table) => {
 		table.increments('id');
@@ -741,6 +787,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -758,6 +805,13 @@ exports.up = async (knex) => {
 		table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
 			.defaultTo('day');
 
+		table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
+			.defaultTo('day');
+
+		table.string('language_alpha2')
+			.references('alpha2')
+			.inTable('languages');
+
 		table.text('description');
 
 		table.integer('duration')
@@ -773,6 +827,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -937,6 +993,8 @@ exports.up = async (knex) => {
 	});
 
 	await knex.schema.createTable('releases_tags', (table) => {
+		table.increments('id');
+
 		table.integer('tag_id', 12)
 			.references('id')
 			.inTable('tags');
@@ -947,9 +1005,13 @@ exports.up = async (knex) => {
 			.inTable('releases')
 			.onDelete('cascade');
 
+		table.integer('actor_id')
+			.references('id')
+			.inTable('actors');
+
 		table.text('original_tag');
 
-		table.unique(['tag_id', 'release_id']);
+		table.unique(['release_id', 'original_tag']);
 		table.index('tag_id');
 		table.index('release_id');
 
@@ -958,12 +1020,7 @@ exports.up = async (knex) => {
 			.defaultTo('scraper');
 	});
 
-	await knex.schema.createTable('releases_search', (table) => {
+	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1));');
-		table.integer('release_id', 16)
-			.references('id')
-			.inTable('releases')
-			.onDelete('cascade');
-	});
 
 	await knex.schema.createTable('movies', (table) => {
 		table.increments('id');
@@ -984,6 +1041,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -1000,6 +1058,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1134,13 +1194,6 @@ exports.up = async (knex) => {
 		table.unique(['movie_id', 'media_id']);
 	});
 
-	await knex.schema.createTable('movies_search', (table) => {
-		table.integer('movie_id', 16)
-			.references('id')
-			.inTable('movies')
-			.onDelete('cascade');
-	});
-
 	await knex.schema.createTable('series', (table) => {
 		table.increments('id');
 
@@ -1159,6 +1212,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -1175,6 +1229,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1288,13 +1344,6 @@ exports.up = async (knex) => {
 		table.unique(['serie_id', 'media_id']);
 	});
 
-	await knex.schema.createTable('series_search', (table) => {
-		table.integer('serie_id', 16)
-			.references('id')
-			.inTable('series')
-			.onDelete('cascade');
-	});
-
 	await knex.schema.createTable('chapters', (table) => {
 		table.increments('id');
 
@@ -1316,6 +1365,8 @@ exports.up = async (knex) => {
 		table.text('title');
 		table.text('description');
 
+		table.datetime('date');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1379,6 +1430,34 @@ exports.up = async (knex) => {
 		table.unique(['tag_id', 'chapter_id']);
 	});
 
+	await knex.schema.createTable('chapters_trailers', (table) => {
+		table.integer('chapter_id')
+			.notNullable()
+			.references('id')
+			.inTable('chapters')
+			.onDelete('cascade');
+
+		table.text('media_id')
+			.notNullable()
+			.references('id')
+			.inTable('media')
+			.onDelete('cascade');
+	});
+
+	await knex.schema.createTable('chapters_teasers', (table) => {
+		table.integer('chapter_id')
+			.notNullable()
+			.references('id')
+			.inTable('chapters')
+			.onDelete('cascade');
+
+		table.text('media_id')
+			.notNullable()
+			.references('id')
+			.inTable('media')
+			.onDelete('cascade');
+	});
+
 	await knex.schema.createTable('users_roles', (table) => {
 		table.string('role')
 			.primary();
@@ -1396,7 +1475,9 @@ exports.up = async (knex) => {
 				{ subject: 'actor', action: 'create' },
 				{ subject: 'actor', action: 'update' },
 				{ subject: 'actor', action: 'delete' },
-				{ plainUrls: true },
+				{ subject: 'actor', action: 'merge' },
+				{ subject: 'sync' },
+				{ subject: 'plainUrls' },
 			]),
 		},
 		{
@@ -1442,6 +1523,8 @@ exports.up = async (knex) => {
 
 		table.specificType('last_ip', 'cidr');
 
+		table.jsonb('settings');
+
 		table.datetime('created_at')
 			.notNullable()
 			.defaultTo(knex.fn.now());
@@ -1660,6 +1743,98 @@ exports.up = async (knex) => {
 			.defaultTo(knex.fn.now());
 	});
 
+	await knex.schema.createTable('feeds', (table) => {
+		table.increments('id');
+
+		table.integer('user_id')
+			.notNullable()
+			.references('id')
+			.inTable('users')
+			.onDelete('cascade');
+
+		table.string('name')
+			.notNullable();
+
+		table.string('slug')
+			.notNullable();
+
+		table.boolean('public');
+		table.boolean('primary');
+
+		table.text('comment');
+		table.json('meta');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_entities', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('entity_id')
+			.notNullable()
+			.references('id')
+			.inTable('entities')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_actors', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('actor_id')
+			.notNullable()
+			.references('id')
+			.inTable('actors')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_tags', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('tag_id')
+			.notNullable()
+			.references('id')
+			.inTable('tags')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
 	await knex.schema.createTable('alerts', (table) => {
 		table.increments('id');
 
@@ -1698,6 +1873,10 @@ exports.up = async (knex) => {
 			.notNullable()
 			.defaultTo(false);
 
+		table.boolean('is_expired')
+			.notNullable()
+			.defaultTo(false);
+
 		table.json('meta');
 		table.text('comment');
 
@@ -1878,6 +2057,31 @@ exports.up = async (knex) => {
 		);
 	});
 
+	await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
+		view.columns('user_id', 'scene_id', 'alert_ids');
+
+		view.as(
+			knex('alerts_scenes')
+				.select(
+					'alerts.user_id',
+					'alerts_scenes.scene_id',
+					knex.raw('array_agg(distinct alerts.id) as alert_ids'),
+					knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
+				)
+				.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
+					joinBuilder
+						.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
+						.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
+				})
+				.where('alerts.is_expired', false)
+				.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
+		);
+	});
+
 	await knex.schema.createTable('notifications', (table) => {
 		table.increments('id');
 
@@ -1901,6 +2105,18 @@ exports.up = async (knex) => {
 			.notNullable()
 			.defaultTo(false);
 
+		table.datetime('seen_at');
+
+		table.boolean('notify')
+			.notNullable()
+			.defaultTo(true);
+
+		table.boolean('email')
+			.notNullable()
+			.defaultTo(false);
+
+		table.datetime('emailed_at');
+
 		table.datetime('created_at')
 			.notNullable()
 			.defaultTo(knex.fn.now());
@@ -2177,12 +2393,20 @@ exports.up = async (knex) => {
 			.inTable('entities');
 	});
 
+	await knex.schema.createTable('sync', (table) => {
+		table.increments('id');
+
+		table.string('domain');
+		table.specificType('item_ids', 'integer array');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	});
+
 	// SEARCH AND SORT
 	await knex.raw(`
-		ALTER TABLE releases_search ADD COLUMN document tsvector;
-		ALTER TABLE movies_search ADD COLUMN document tsvector;
-		ALTER TABLE series_search ADD COLUMN document tsvector;
-
 		/* allow scenes without dates to be mixed inbetween scenes with dates */
 		ALTER TABLE releases
 		ADD COLUMN effective_date timestamptz
@@ -2200,19 +2424,12 @@ exports.up = async (knex) => {
 	// INDEXES
 	await knex.raw(`
 		CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id);
-		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
+		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
 
 		CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL;
-
-		CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
-		CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
-		CREATE INDEX releases_search_index ON releases_search USING GIN (document);
-		CREATE INDEX movies_search_index ON movies_search USING GIN (document);
-		CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
-		CREATE INDEX series_search_index ON series_search USING GIN (document);
 	`);
 
 	// FUNCTIONS
@@ -2275,7 +2492,8 @@ exports.up = async (knex) => {
 				stashes.id as stash_id,
 				COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes,
 				COUNT(DISTINCT stashes_movies)::integer as stashed_movies,
-				COUNT(DISTINCT stashes_actors)::integer as stashed_actors
+				COUNT(DISTINCT stashes_actors)::integer as stashed_actors,
+				MAX(GREATEST(stashes_scenes.created_at, stashes_movies.created_at, stashes_actors.created_at)) as last_stash_at
 			FROM stashes
 			LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id
 			LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id
@@ -2372,6 +2590,8 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS chapters_tags CASCADE;
 		DROP TABLE IF EXISTS chapters_posters CASCADE;
 		DROP TABLE IF EXISTS chapters_photos CASCADE;
+		DROP TABLE IF EXISTS chapters_trailers CASCADE;
+		DROP TABLE IF EXISTS chapters_teasers CASCADE;
 
 		DROP TABLE IF EXISTS banners_tags CASCADE;
 		DROP TABLE IF EXISTS banners CASCADE;
@@ -2381,6 +2601,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 
 		DROP TABLE IF EXISTS batches CASCADE;
 		DROP TABLE IF EXISTS fingerprints_types CASCADE;
+		DROP TABLE IF EXISTS sync CASCADE;
 
 		DROP TABLE IF EXISTS actors_avatars CASCADE;
 		DROP TABLE IF EXISTS actors_photos CASCADE;
@@ -2412,8 +2633,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS sites CASCADE;
 		DROP TABLE IF EXISTS studios CASCADE;
 		DROP TABLE IF EXISTS media_sfw CASCADE;
+		DROP TABLE IF EXISTS media_biometrics CASCADE;
 		DROP TABLE IF EXISTS media CASCADE;
 		DROP TABLE IF EXISTS countries CASCADE;
+		DROP TABLE IF EXISTS languages CASCADE;
 		DROP TABLE IF EXISTS networks CASCADE;
 
 		DROP TABLE IF EXISTS entities_types CASCADE;
@@ -2426,6 +2649,11 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS stashes_actors CASCADE;
 		DROP TABLE IF EXISTS stashes CASCADE;
 
+		DROP TABLE IF EXISTS feeds_entities CASCADE;
+		DROP TABLE IF EXISTS feeds_actors CASCADE;
+		DROP TABLE IF EXISTS feeds_tags CASCADE;
+		DROP TABLE IF EXISTS feeds CASCADE;
+
 		DROP TABLE IF EXISTS alerts_scenes CASCADE;
 		DROP TABLE IF EXISTS alerts_actors CASCADE;
 		DROP TABLE IF EXISTS alerts_tags CASCADE;

migrations/20260203052449_scene_attributes.js

@@ -1,19 +0,0 @@
-exports.up = async (knex) => {
-	await knex.schema.alterTable('releases', (table) => {
-		table.json('attributes');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.json('attributes');
-	});
-};
-
-exports.down = async (knex) => {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('attributes');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.dropColumn('attributes');
-	});
-};

migrations/20260206034715_raw_releases.js

@@ -1,11 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('alt_descriptions');
-	});
-};

migrations/20260207034922_chapter_details.js

@@ -1,58 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('chapters', (table) => {
-		table.datetime('date');
-	});
-
-	await knex.schema.createTable('chapters_trailers', (table) => {
-		table.integer('chapter_id')
-			.notNullable()
-			.references('id')
-			.inTable('chapters')
-			.onDelete('cascade');
-
-		table.text('media_id')
-			.notNullable()
-			.references('id')
-			.inTable('media')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.createTable('chapters_teasers', (table) => {
-		table.integer('chapter_id')
-			.notNullable()
-			.references('id')
-			.inTable('chapters')
-			.onDelete('cascade');
-
-		table.text('media_id')
-			.notNullable()
-			.references('id')
-			.inTable('media')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.alterTable('entities', (table) => {
-		table.string('name_stylized');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('chapters', (table) => {
-		table.dropColumn('date');
-	});
-
-	await knex.schema.alterTable('entities', (table) => {
-		table.dropColumn('name_stylized');
-	});
-
-	await knex.schema.dropTable('chapters_trailers');
-	await knex.schema.dropTable('chapters_teasers');
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.dropColumn('alt_descriptions');
-	});
-};

migrations/20260208044729_random_sfw_improvements.js

@@ -1,28 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createMaterializedView('media_sfw', (view) => {
-		view.as(knex('media').select('id').where('is_sfw', true));
-	});
-
-	await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
-
-	await knex.raw(`
-		CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media_sfw
-			ORDER BY random()
-			LIMIT 1;
-		$$ LANGUAGE sql STABLE;
-	`);
-};
-
-exports.down = async function(knex) {
-	await knex.raw(`
-		CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media
-			WHERE is_sfw = true
-			ORDER BY random()
-			LIMIT 1;
-		$$ LANGUAGE sql STABLE;
-	`);
-
-	await knex.schema.dropMaterializedView('media_sfw');
-};

migrations/20260222055254_unique_origin_tags.js

@@ -1,31 +0,0 @@
-exports.up = async (knex) => {
-	// dedupe
-	await knex.raw(`
-		DELETE
-		FROM releases_tags
-		WHERE ctid IN
-		(
-			SELECT ctid
-			FROM(
-				SELECT
-					*,
-					ctid,
-					row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
-				FROM releases_tags
-			)s
-			WHERE row_number >= 2
-		)
-	`);
-
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.increments('id');
-		table.unique(['release_id', 'original_tag']);
-	});
-};
-
-exports.down = async (knex) => {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.dropColumn('id');
-		table.dropUnique(['release_id', 'original_tag']);
-	});
-};

migrations/20260301042453_foot_float.js

@@ -1,23 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('actors', (table) => {
-		table.decimal('foot')
-			.alter();
-	});
-
-	await knex.schema.alterTable('actors_profiles', (table) => {
-		table.decimal('foot')
-			.alter();
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('actors', (table) => {
-		table.integer('foot')
-			.alter();
-	});
-
-	await knex.schema.alterTable('actors_profiles', (table) => {
-		table.integer('foot')
-			.alter();
-	});
-};

migrations/20260302222545_series_alt_descriptions.js

@@ -1,13 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('series', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-		table.json('attributes');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('series', (table) => {
-		table.dropColumn('alt_descriptions');
-		table.dropColumn('attributes');
-	});
-};

migrations/20260304020542_scene_actor_tags.js

@@ -1,21 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.integer('actor_id')
-			.references('id')
-			.inTable('actors');
-
-		table.dropUnique(['tag_id', 'release_id']);
-	});
-
-	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.dropColumn('actor_id');
-
-		table.unique(['tag_id', 'release_id']);
-	});
-
-	await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
-};

migrations/20260402051404_scene_language_production_date_precision.js

@@ -1,27 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createTable('languages', (table) => {
-		table.string('alpha2')
-			.primary();
-
-		table.text('name');
-		table.text('name_native');
-	});
-
-	await knex.schema.alterTable('releases', (table) => {
-		table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
-			.defaultTo('day');
-
-		table.string('language_alpha2')
-			.references('alpha2')
-			.inTable('languages');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('production_date_precision');
-		table.dropColumn('language_alpha2');
-	});
-
-	await knex.schema.dropTable('languages');
-};

migrations/20260403231603_feeds.js

@@ -1,100 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createTable('feeds', (table) => {
-		table.increments('id');
-
-		table.integer('user_id')
-			.notNullable()
-			.references('id')
-			.inTable('users')
-			.onDelete('cascade');
-
-		table.string('name')
-			.notNullable();
-
-		table.string('slug')
-			.notNullable();
-
-		table.boolean('public');
-		table.boolean('primary');
-
-		table.text('comment');
-		table.json('meta');
-
-		table.datetime('created_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-	});
-
-	await knex.schema.createTable('feeds_entities', (table) => {
-		table.increments('id');
-
-		table.integer('feed_id')
-			.notNullable()
-			.references('id')
-			.inTable('feeds')
-			.onDelete('cascade');
-
-		table.integer('entity_id')
-			.notNullable()
-			.references('id')
-			.inTable('entities')
-			.onDelete('cascade');
-
-		table.text('comment');
-
-		table.datetime('created_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-	});
-
-	await knex.schema.createTable('feeds_actors', (table) => {
-		table.increments('id');
-
-		table.integer('feed_id')
-			.notNullable()
-			.references('id')
-			.inTable('feeds')
-			.onDelete('cascade');
-
-		table.integer('actor_id')
-			.notNullable()
-			.references('id')
-			.inTable('actors')
-			.onDelete('cascade');
-
-		table.text('comment');
-
-		table.datetime('created_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-	});
-
-	await knex.schema.createTable('feeds_tags', (table) => {
-		table.increments('id');
-
-		table.integer('feed_id')
-			.notNullable()
-			.references('id')
-			.inTable('feeds')
-			.onDelete('cascade');
-
-		table.integer('tag_id')
-			.notNullable()
-			.references('id')
-			.inTable('tags')
-			.onDelete('cascade');
-
-		table.text('comment');
-
-		table.datetime('created_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.dropTable('feeds_tags');
-	await knex.schema.dropTable('feeds_actors');
-	await knex.schema.dropTable('feeds_entities');
-	await knex.schema.dropTable('feeds');
-};

migrations/20260520044355_actors_unique.js

@@ -1,59 +0,0 @@
-exports.up = async function(knex) {
-	await knex.raw(`
-		DROP INDEX unique_actor_slugs;
-		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
-	`);
-
-	await knex.schema.alterTable('actors', (table) => {
-		table.boolean('allow_global_match');
-		table.text('comment');
-	});
-
-	await knex('users_roles')
-		.update('abilities', JSON.stringify([
-			{ subject: 'scene', action: 'create' },
-			{ subject: 'scene', action: 'update' },
-			{ subject: 'scene', action: 'delete' },
-			{ subject: 'actor', action: 'create' },
-			{ subject: 'actor', action: 'update' },
-			{ subject: 'actor', action: 'delete' },
-			{ subject: 'actor', action: 'merge' },
-			{ plainUrls: true },
-		]))
-		.where('role', 'admin');
-};
-
-exports.down = async function(knex) {
-	const dupes = await knex('actors')
-		.select('name', 'slug', knex.raw('count(*) as count'))
-		.whereNull('entity_id')
-		.groupBy('name', 'slug')
-		.havingRaw('count(*) > 1')
-		.orderBy('count', 'desc');
-
-	if (dupes.length > 0) {
-		console.log('DUPES\n', dupes.map((actor) => `${actor.name} ${actor.slug} ${actor.count}`).join('\n'));
-	}
-
-	await knex.raw(`
-		DROP INDEX unique_actor_slugs;
-		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
-	`);
-
-	await knex.schema.alterTable('actors', (table) => {
-		table.dropColumn('allow_global_match');
-		table.dropColumn('comment');
-	});
-
-	await knex('users_roles')
-		.update('abilities', JSON.stringify([
-			{ subject: 'scene', action: 'create' },
-			{ subject: 'scene', action: 'update' },
-			{ subject: 'scene', action: 'delete' },
-			{ subject: 'actor', action: 'create' },
-			{ subject: 'actor', action: 'update' },
-			{ subject: 'actor', action: 'delete' },
-			{ plainUrls: true },
-		]))
-		.where('role', 'admin');
-};

migrations/20260608053154_sync_abilities.js

@@ -1,87 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createTable('sync', (table) => {
-		table.increments('id');
-
-		table.string('domain');
-		table.specificType('item_ids', 'integer array');
-
-		table.text('comment');
-
-		table.datetime('created_at')
-			.defaultTo(knex.fn.now());
-	});
-
-	await knex('users_roles')
-		.update('abilities', JSON.stringify([
-			{ subject: 'scene', action: 'create' },
-			{ subject: 'scene', action: 'update' },
-			{ subject: 'scene', action: 'delete' },
-			{ subject: 'actor', action: 'create' },
-			{ subject: 'actor', action: 'update' },
-			{ subject: 'actor', action: 'delete' },
-			{ subject: 'actor', action: 'merge' },
-			{ subject: 'sync' },
-			{ subject: 'plainUrls' },
-		]))
-		.where('role', 'admin');
-
-	await knex.raw(`
-		DROP TABLE IF EXISTS releases_search CASCADE;
-		DROP TABLE IF EXISTS movies_search CASCADE;
-		DROP TABLE IF EXISTS series_search CASCADE;
-
-		DROP TABLE IF EXISTS releases_search_results CASCADE;
-		DROP TABLE IF EXISTS movies_search_results CASCADE;
-	`);
-};
-
-exports.down = async function(knex) {
-	await knex.schema.dropTable('sync');
-
-	await knex('users_roles')
-		.update('abilities', JSON.stringify([
-			{ subject: 'scene', action: 'create' },
-			{ subject: 'scene', action: 'update' },
-			{ subject: 'scene', action: 'delete' },
-			{ subject: 'actor', action: 'create' },
-			{ subject: 'actor', action: 'update' },
-			{ subject: 'actor', action: 'delete' },
-			{ subject: 'actor', action: 'merge' },
-			{ plainUrls: true },
-		]))
-		.where('role', 'admin');
-
-	await knex.schema.createTable('releases_search', (table) => {
-		table.integer('release_id', 16)
-			.references('id')
-			.inTable('releases')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.createTable('movies_search', (table) => {
-		table.integer('movie_id', 16)
-			.references('id')
-			.inTable('movies')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.createTable('series_search', (table) => {
-		table.integer('serie_id', 16)
-			.references('id')
-			.inTable('series')
-			.onDelete('cascade');
-	});
-
-	await knex.raw(`
-		ALTER TABLE releases_search ADD COLUMN document tsvector;
-		ALTER TABLE movies_search ADD COLUMN document tsvector;
-		ALTER TABLE series_search ADD COLUMN document tsvector;
-
-		CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
-		CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
-		CREATE INDEX releases_search_index ON releases_search USING GIN (document);
-		CREATE INDEX movies_search_index ON movies_search USING GIN (document);
-		CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
-		CREATE INDEX series_search_index ON series_search USING GIN (document);
-	`);
-};

migrations/20260704023524_biometrics.js

@@ -1,35 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createTable('media_biometrics', (table) => {
-		table.increments('id');
-
-		table.string('media_id', 21)
-			.references('id')
-			.inTable('media')
-			.notNullable()
-			.onDelete('cascade');
-
-		table.integer('width');
-		table.integer('height');
-
-		table.integer('face_index')
-			.notNullable()
-			.defaultTo(0);
-
-		table.json('biometrics');
-		table.specificType('embedding', 'vector(1024)');
-
-		table.datetime('updated_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-
-		table.datetime('created_at')
-			.notNullable()
-			.defaultTo(knex.fn.now());
-
-		table.unique(['media_id', 'face_index']);
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.dropTable('media_biometrics');
-};

migrations/20260726032348_stashes_archive.js

@@ -0,0 +1,15 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('stashes', (table) => {
+		table.boolean('archived')
+			.notNullable()
+			.defaultTo(false);
+
+		table.check('NOT ("primary" AND archived)', [], 'stashes_not_primary_and_archived');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('stashes', (table) => {
+		table.dropColumn('archived');
+	});
+};

migrations/20260823020033_actors_date_precision.js

@@ -0,0 +1,33 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('actors', (table) => {
+		table.string('date_of_birth_precision')
+			.notNullable()
+			.defaultTo('day');
+
+		table.string('date_of_death_precision')
+			.notNullable()
+			.defaultTo('day');
+	});
+
+	await knex.schema.alterTable('actors_profiles', (table) => {
+		table.string('date_of_birth_precision')
+			.notNullable()
+			.defaultTo('day');
+
+		table.string('date_of_death_precision')
+			.notNullable()
+			.defaultTo('day');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('actors', (table) => {
+		table.dropColumn('date_of_birth_precision');
+		table.dropColumn('date_of_death_precision');
+	});
+
+	await knex.schema.alterTable('actors_profiles', (table) => {
+		table.dropColumn('date_of_birth_precision');
+		table.dropColumn('date_of_death_precision');
+	});
+};

migrations/20260922165429_anonymous_actors.js

@@ -0,0 +1,24 @@
+exports.up = async function(knex) {
+	// we want to be able to assign unidentified actors
+	await knex.raw(`
+		ALTER TABLE releases_actors
+		ALTER COLUMN actor_id DROP NOT NULL
+	`);
+
+	await knex.schema.alterTable('releases_actors', (table) => {
+		table.increments('id');
+		table.string('gender');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.raw(`
+		ALTER TABLE releases_actors
+		ALTER COLUMN actor_id SET NOT NULL
+	`);
+
+	await knex.schema.alterTable('releases_actors', (table) => {
+		table.dropColumn('id');
+		table.dropColumn('gender');
+	});
+};

package.json

@@ -1,16 +1,13 @@
 {
     "name": "traxxx",
-    "version": "1.254.12",
+    "version": "1.257.6",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
+    "engines": {
+        "node": ">=26"
+    },
     "scripts": {
         "start": "node -r source-map-support/register src/init.js",
-        "webpack": "webpack --env=production --mode=production",
-        "webpack-dev": "webpack --env=development --mode=development",
-        "webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
-        "babel": "babel src --source-maps -d dist",
-        "babel-watch": "babel src -w --source-maps -d dist",
-        "build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
         "eslint": "eslint src/",
         "eslint-watch": "esw --watch src/",
         "knex": "knex",
@@ -38,135 +35,60 @@
     "author": "DebaucheryLibrarian",
     "license": "ISC",
     "devDependencies": {
-        "@babel/cli": "^7.23.4",
-        "@babel/core": "^7.23.3",
-        "@babel/eslint-parser": "^7.23.3",
-        "@babel/plugin-proposal-optional-chaining": "^7.8.3",
-        "@babel/preset-env": "^7.23.3",
-        "@babel/register": "^7.22.15",
-        "autoprefixer": "^10.4.16",
-        "babel-loader": "^9.1.3",
-        "babel-preset-airbnb": "^5.0.0",
-        "css-loader": "^6.8.1",
         "eslint": "^8.54.0",
-        "eslint-config-airbnb": "^19.0.4",
         "eslint-config-airbnb-base": "^15.0.0",
         "eslint-plugin-import": "^2.29.0",
-        "eslint-plugin-jsx-a11y": "^6.8.0",
+        "eslint-watch": "^8.0.0"
-        "eslint-plugin-react": "^7.33.2",
-        "eslint-plugin-vue": "^9.18.1",
-        "eslint-watch": "^8.0.0",
-        "eslint-webpack-plugin": "^4.0.1",
-        "mini-css-extract-plugin": "^2.7.6",
-        "node-sass": "^9.0.0",
-        "postcss-loader": "^7.3.3",
-        "raw-loader": "^4.0.2",
-        "sass-loader": "^13.3.2",
-        "style-loader": "^3.3.3",
-        "vue-loader": "^17.3.1",
-        "webpack": "^5.89.0",
-        "webpack-cli": "^5.1.4"
     },
     "dependencies": {
         "@aws-sdk/client-s3": "^3.458.0",
-        "@aws-sdk/credential-provider-ini": "^3.458.0",
         "@aws-sdk/lib-storage": "^3.458.0",
-        "@casl/ability": "^6.5.0",
+        "@faker-js/faker": "^10.6.0",
-        "@faker-js/faker": "^8.3.1",
+        "@noble/hashes": "^2.4.0",
-        "@graphile-contrib/pg-order-by-related": "^1.0.0",
-        "@graphile-contrib/pg-simplify-inflector": "^6.1.0",
-        "@graphile/pg-aggregates": "^0.1.1",
         "@rsc-parser/react-client": "^1.1.2",
+        "@tensorflow/tfjs-core": "4.22.0",
         "@tensorflow/tfjs-node": "^4.22.0",
         "@vladmandic/human": "^3.3.6",
-        "acorn": "^8.11.2",
-        "array-equal": "^1.0.2",
-        "babel-polyfill": "^6.26.0",
-        "bhttp": "^1.2.8",
-        "blake2": "^5.0.0",
         "bluebird": "^3.7.2",
-        "body-parser": "^1.20.2",
         "bottleneck": "^2.19.5",
         "casual": "^1.6.2",
-        "cheerio": "^1.0.0-rc.12",
         "cli-confirm": "^1.0.1",
-        "clipboardy": "^2.3.0",
+        "config": "^5.0.1",
-        "cloudscraper": "^4.6.0",
+        "convert": "^7.0.2",
-        "config": "^3.3.9",
-        "connect-session-knex": "^4.0.0",
-        "convert": "^4.14.0",
-        "cookie": "^0.6.0",
-        "css": "^3.0.0",
-        "csv-parse": "^6.1.0",
         "csv-stringify": "^6.4.4",
-        "date-fns": "^2.30.0",
+        "date-fns": "^4.4.0",
-        "dayjs": "^1.11.10",
         "dompurify": "^3.0.6",
-        "ejs": "^3.1.9",
+        "escape-string-regexp": "^5.0.0",
-        "escape-string-regexp": "^4.0.0",
-        "express": "^4.18.2",
-        "express-promise-router": "^4.1.1",
-        "express-query-boolean": "^2.0.0",
-        "express-react-views": "^0.11.0",
-        "express-session": "^1.17.3",
-        "face-api.js": "^0.22.2",
-        "file-type": "^18.7.0",
         "fluent-ffmpeg": "^2.1.3",
         "fs-extra": "^11.1.1",
-        "graphile-build": "^4.14.0",
-        "graphile-utils": "^4.14.0",
-        "graphql": "^15.8.0",
         "html-entities": "^2.4.0",
-        "https-proxy-agent": "^7.0.5",
+        "inquirer": "^14.2.5",
-        "iconv-lite": "^0.6.3",
-        "inquirer": "^8.2.6",
         "inspector-api": "^1.4.8",
-        "jsdom": "^23.0.1",
+        "jsdom": "^31.0.0",
         "knex": "^3.0.1",
         "knex-migrate": "^1.7.4",
-        "longjohn": "^0.2.12",
+        "mime": "^4.1.2",
-        "manticoresearch": "^4.0.0",
-        "merge-anything": "^5.1.7",
-        "mime": "^3.0.0",
-        "mitt": "^3.0.1",
         "moment": "^2.29.4",
-        "mysql": "^2.18.1",
+        "nanoid": "^6.0.2",
-        "nanoid": "^3.3.7",
         "node-fetch": "^3.3.2",
-        "npm": "^11.7.0",
+        "object-merge-advanced": "^14.2.5",
-        "object-merge-advanced": "^12.1.0",
         "object.omit": "^3.0.0",
         "pg": "^8.11.3",
-        "playwright": "^1.57.0",
-        "playwright-core": "^1.57.0",
-        "postgraphile": "^4.14.0",
-        "postgraphile-plugin-connection-filter": "^2.3.0",
         "promise-task-queue": "^1.2.0",
-        "prop-types": "^15.8.1",
+        "redis": "^6.3.1",
-        "puppeteer": "^24.4.0",
+        "sharp": "^0.35.5",
-        "puppeteer-extra": "^3.3.6",
-        "puppeteer-extra-plugin-stealth": "^2.11.2",
-        "redis": "^4.6.11",
-        "sharp": "^0.33.0-rc.2",
-        "showdown": "^2.1.0",
         "source-map-support": "^0.5.21",
-        "template-format": "^1.2.5",
+        "template-format": "^2.0.0",
-        "tippy.js": "^6.3.7",
+        "undici": "^8.12.4",
-        "tough-cookie": "^4.1.3",
+        "unprint": "^1.0.0-beta.2",
-        "tunnel": "0.0.6",
+        "why-is-node-running": "^3.2.3",
-        "ua-parser-js": "^1.0.37",
-        "undici": "^7.24.7",
-        "unprint": "^0.19.20",
-        "url-pattern": "^1.0.3",
-        "v-tooltip": "^2.1.3",
-        "video.js": "^8.6.1",
-        "videojs-vr": "^2.0.0",
-        "vue": "^3.3.9",
-        "vue-router": "^4.2.5",
-        "vuex": "^4.1.0",
-        "why-is-node-running": "^2.2.2",
         "winston": "^3.11.0",
-        "winston-daily-rotate-file": "^4.7.1",
+        "winston-daily-rotate-file": "^5.0.0",
-        "yargs": "^17.7.2"
+        "yargs": "^18.2.0"
+    },
+    "allowScripts": {
+        "sharp": true,
+        "@tensorflow/tfjs-node": true,
+        "core-js": false
     }
 }

seeds/00_tags.js

@@ -2,6 +2,10 @@ const upsert = require('../src/utils/upsert');
 const slugify = require('../src/utils/slugify');
 
 const groups = [
+	{
+		slug: 'accessories',
+		name: 'Accessories',
+	},
 	{
 		slug: 'age',
 		name: 'Age',
@@ -344,10 +348,31 @@ const tags = [
 		slug: 'brunette',
 		group: 'hair',
 	},
+	{
+		name: 'body writing',
+		slug: 'body-writing',
+		description: 'Grab a marker or a lipstick and use your body as a canvas. Label all your holes and make it clear what they should call you.',
+	},
+	{
+		name: 'boots',
+		slug: 'boots',
+		group: 'clothing',
+	},
+	{
+		name: 'pleasers',
+		slug: 'pleasers',
+		group: 'clothing',
+	},
 	{
 		name: 'bukkake',
 		slug: 'bukkake',
-		description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
+		description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). If the mouth filled up directly or the cum is collected and drunk from a cup, it can be referred to as a [gokkun](/tag/gokkun).',
+		group: 'finish',
+	},
+	{
+		name: 'gokkun',
+		slug: 'gokkun',
+		description: 'Drinking cum from a group of three or more men, either ejaculated directly into the mouth or collected and drank from a cup, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). It is essentially the swallowing equivalent of a [bukkake](/tag/bukkake).',
 		group: 'finish',
 	},
 	{
@@ -406,6 +431,10 @@ const tags = [
 		name: 'cum',
 		slug: 'cum',
 	},
+	{
+		name: 'cum covered fucking',
+		slug: 'ccf',
+	},
 	{
 		name: 'cum drunk',
 		slug: 'cum-drunk',
@@ -688,7 +717,12 @@ const tags = [
 	{
 		name: 'gangbang',
 		slug: 'gangbang',
-		description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
+		description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a [reverse gangbang](/tag/reverse-gangbang), multiple women fuck one man.', /* eslint-disable-line max-len */
+		group: 'group',
+	},
+	{
+		name: 'reverse gangbang',
+		slug: 'reverse-gangbang',
 		group: 'group',
 	},
 	{
@@ -762,6 +796,11 @@ const tags = [
 		slug: 'high-heels',
 		group: 'clothing',
 	},
+	{
+		name: 'hotel',
+		slug: 'hotel',
+		group: 'location',
+	},
 	{
 		name: 'humiliation',
 		slug: 'humiliation',
@@ -1259,6 +1298,22 @@ const tags = [
 		slug: 'toy-dp',
 		description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
 	},
+	{
+		name: 'toy DVP',
+		slug: 'toy-dvp',
+	},
+	{
+		name: 'toy DAP',
+		slug: 'toy-dap',
+	},
+	{
+		name: 'toy TAP',
+		slug: 'toy-tap',
+	},
+	{
+		name: 'toy TVP',
+		slug: 'toy-tvp',
+	},
 	{
 		name: 'finger DP',
 		slug: 'finger-dp',
@@ -1267,6 +1322,16 @@ const tags = [
 		name: 'transsexual',
 		slug: 'transsexual',
 	},
+	{
+		name: 'trans man',
+		slug: 'trans-man',
+		description: 'A man who was assigned female at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
+	},
+	{
+		name: 'trans woman',
+		slug: 'trans-woman',
+		description: 'woman man who was assigned male at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
+	},
 	{
 		name: 'DA triple penetration',
 		slug: 'da-tp',
@@ -1460,6 +1525,16 @@ const tags = [
 		name: 'pregnant',
 		slug: 'pregnant',
 	},
+	{
+		name: 'collar',
+		slug: 'collar',
+		group: 'accessories',
+	},
+	{
+		name: 'choker',
+		slug: 'choker',
+		group: 'accessories',
+	},
 ];
 
 const aliases = [
@@ -1633,6 +1708,10 @@ const aliases = [
 		name: 'mmf',
 		for: 'mfm',
 	},
+	{
+		name: 'threesome mmf',
+		for: 'mfm',
+	},
 	{
 		name: 'fmf',
 		for: 'mff',
@@ -1706,6 +1785,10 @@ const aliases = [
 		name: 'busty - big boobs',
 		for: 'big-boobs',
 	},
+	{
+		name: 'big breasts',
+		for: 'big-boobs',
+	},
 	{
 		name: 'busty: big beautiful breast',
 		for: 'big-boobs',
@@ -3130,6 +3213,22 @@ const aliases = [
 		name: 'fist',
 		for: 'fisting',
 	},
+	{
+		name: 'boot',
+		for: 'boots',
+	},
+	{
+		name: 'pleaser boots',
+		for: 'pleasers',
+	},
+	{
+		name: 'pleaser shoes',
+		for: 'pleasers',
+	},
+	{
+		name: 'pleaser heels',
+		for: 'pleasers',
+	},
 ];
 
 const priorities = [ // higher index is higher priority

seeds/01_networks.js

@@ -172,7 +172,7 @@ const networks = [
 	{
 		slug: 'familystrokes',
 		name: 'Family Strokes',
-		url: 'https://www.familystrokes.com/',
+		url: 'https://www.familystrokes.com',
 		parent: 'paperstreetmedia',
 		parameters: {
 			endpoint: 'familybundle',
@@ -284,6 +284,7 @@ const networks = [
 			areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
 			blockId: 114458,
 			scene: 'https://aziani.com',
+			api: 'https://azianistudios.com/tour_api.php',
 			cdn: 'https://c75c0c3063.mjedge.net',
 		},
 	},
@@ -593,6 +594,11 @@ const networks = [
 			concurrency: 1,
 		},
 	},
+	{
+		slug: 'ksm',
+		name: 'KSM',
+		hasLogo: false,
+	},
 	{
 		slug: 'letsdoeit',
 		name: 'LetsDoeIt',
@@ -777,6 +783,11 @@ const networks = [
 			parentSession: false,
 		},
 	},
+	{
+		slug: 'rsghmedia',
+		name: 'RSGH Media',
+		hasLogo: false,
+	},
 	{
 		slug: 'score',
 		name: 'SCORE',

seeds/02_sites.js

@@ -6027,56 +6027,41 @@ const sites = [
 		slug: 'sexuallybroken',
 		name: 'Sexually Broken',
 		alias: ['seb'],
-		url: 'https://www.sexuallybroken.com',
+		url: 'https://www.insex.com/is/home.php?d=sexuallybroken.com',
 		tags: ['bdsm'],
 		parent: 'insex',
-		parameters: {
-			latest: 'https://www.sexuallybroken.com/sb',
-		},
 	},
 	{
 		slug: 'infernalrestraints',
 		name: 'Infernal Restraints',
 		alias: ['infr'],
-		url: 'https://www.infernalrestraints.com',
+		url: 'https://www.insex.com/is/home.php?d=infernalrestraints.com',
 		tags: ['bdsm'],
 		parent: 'insex',
-		parameters: {
-			latest: 'https://www.infernalrestraints.com/ir',
-		},
 	},
 	{
 		slug: 'hardtied',
 		name: 'Hardtied',
 		alias: ['ht'],
-		url: 'https://www.hardtied.com',
+		url: 'https://www.insex.com/is/home.php?d=hardtied.com',
 		tags: ['bdsm'],
 		parent: 'insex',
-		parameters: {
-			latest: 'https://www.hardtied.com/ht',
-		},
 	},
 	{
 		slug: 'realtimebondage',
 		name: 'Real Time Bondage',
 		alias: ['rtb'],
-		url: 'https://www.realtimebondage.com',
+		url: 'https://www.insex.com/is/home.php?d=realtimebondage.com',
 		tags: ['bdsm', 'live'],
 		parent: 'insex',
-		parameters: {
-			latest: 'https://www.realtimebondage.com/rtb',
-		},
 	},
 	{
 		slug: 'topgrl',
 		name: 'TopGrl',
 		alias: ['tg'],
-		url: 'https://www.topgrl.com',
+		url: 'https://www.insex.com/is/home.php?d=topgrl.com',
 		tags: ['bdsm', 'femdom'],
 		parent: 'insex',
-		parameters: {
-			latest: 'https://www.topgrl.com/tg',
-		},
 	},
 	{
 		slug: 'paintoy',
@@ -6127,7 +6112,7 @@ const sites = [
 	{
 		slug: 'blackforwife',
 		name: 'Black for Wife',
-		url: 'https://www.blackforwife.com',
+		url: 'https://www.blackforwife.org',
 		parent: 'jayrock',
 		parameters: {
 			useGamma: true,
@@ -6141,6 +6126,12 @@ const sites = [
 		name: 'Jax Slayher',
 		url: 'https://jaxslayher.com',
 	},
+	// JENNIFER WHITE
+	{
+		slug: 'jenniferwhite',
+		name: 'Jennifer White',
+		url: 'https://www.jenniferwhite.com',
+	},
 	// JESSE LOADS MONSTER FACIALS
 	{
 		slug: 'jesseloadsmonsterfacials',
@@ -7379,6 +7370,16 @@ const sites = [
 			studioId: 1214,
 		},
 	},
+	{
+		name: 'Nick Morris Studio',
+		slug: 'nickmorrisstudio',
+		url: 'https://www.analvids.com/studios/nick_morris_studio',
+		parent: 'analvids',
+		hasLogo: false,
+		parameters: {
+			studioId: 1038,
+		},
+	},
 	// LETS DOE IT
 	{
 		name: 'The White Boxxx',
@@ -7698,6 +7699,7 @@ const sites = [
 		url: 'https://nicoledoshi.manyvids.com',
 		hasLogo: false,
 		parent: 'manyvids',
+		delete: true,
 	},
 	// MARISKA X
 	{
@@ -8500,6 +8502,33 @@ const sites = [
 			id: 'hookup-pad',
 		},
 	},
+	{
+		slug: 'thepickupvan',
+		name: 'The Pickup Van',
+		url: 'https://www.mylf.com/series/the-pickup-van',
+		parent: 'mylf',
+		parameters: {
+			id: 'the-pickup-van',
+		},
+	},
+	{
+		slug: 'motherfucker',
+		name: 'Motherfucker',
+		url: 'https://www.mylf.com/series/motherfucker',
+		parent: 'mylf',
+		parameters: {
+			id: 'motherfucker',
+		},
+	},
+	{
+		slug: 'bluecollarbabes',
+		name: 'Blue Collar Babes',
+		url: 'https://www.mylf.com/series/blue-collar-babes',
+		parent: 'mylf',
+		parameters: {
+			id: 'blue-collar-babes',
+		},
+	},
 	/* premium */
 	{
 		slug: 'analmom',
@@ -8534,12 +8563,9 @@ const sites = [
 	{
 		slug: 'mylfdom',
 		name: 'MYLF Dom',
-		url: 'https://www.mylfdom.com',
+		url: 'https://www.mylf.com/series/mylfdom-ts',
 		tags: ['bdsm', 'milf'],
 		parent: 'mylf',
-		parameters: {
-			endpoint: 'mylf_dom',
-		},
 	},
 	// NAUGHTY AMERICA
 	{
@@ -11097,6 +11123,59 @@ const sites = [
 		slug: 'rickysroom',
 		url: 'https://rickysroom.com',
 	},
+	// RSGH MEDIA
+	{
+		name: 'Family Therapy XXX',
+		slug: 'familytherapyxxx',
+		url: 'https://familytherapyxxx.com',
+		tags: ['family'],
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Mom Comes First',
+		slug: 'momcomesfirst',
+		url: 'https://momcomesfirst.com',
+		tags: ['family'],
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Perfect Girlfriend',
+		slug: 'perfectgirlfriend',
+		url: 'https://perfectgirlfriend.com',
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Anal Therapy XXX',
+		slug: 'analtherapyxxx',
+		url: 'https://analtherapyxxx.com',
+		tags: ['anal'],
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Cock Ninja Studios',
+		slug: 'cockninjastudios',
+		url: 'https://cockninjastudios.com',
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Wife Loves Black',
+		slug: 'wifelovesblack',
+		url: 'https://wifelovesblack.com',
+		parent: 'rsghmedia',
+		independent: true,
+	},
+	{
+		name: 'Teen Loves Black',
+		slug: 'teenlovesblack',
+		url: 'https://teenlovesblack.com',
+		parent: 'rsghmedia',
+		independent: true,
+	},
 	// SCORE
 	{
 		name: '18 Eighteen',
@@ -12327,13 +12406,6 @@ const sites = [
 		url: 'https://www.teamskeet.com/series/bad-milfs',
 		parent: 'teamskeet',
 	},
-	{
-		slug: 'blackstepdad',
-		name: 'Black StepDad',
-		url: 'https://www.teamskeet.com/series/black-stepdad',
-		alias: ['bsd'],
-		parent: 'teamskeet',
-	},
 	{
 		slug: 'exxxtrasmall',
 		name: 'Exxxtra Small',
@@ -12713,6 +12785,44 @@ const sites = [
 		url: 'https://www.teamskeet.com/series/passport-bros',
 		parent: 'teamskeet',
 	},
+	{
+		slug: 'underthetable',
+		name: 'Under the Table',
+		url: 'https://www.teamskeet.com/series/under-the-table',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'mydirtyuncle',
+		name: 'My Dirty Uncle',
+		url: 'https://www.teamskeet.com/series/my-dirty-uncle',
+		tags: ['family'],
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'mormongirlz',
+		name: 'Mormon Girlz',
+		url: 'https://www.teamskeet.com/series/mormon-girlz',
+		tags: ['religion'],
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'breedingmaterial',
+		name: 'Breeding Material',
+		url: 'https://www.teamskeet.com/series/breeding-material',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'teamskeetsingles',
+		name: 'TeamSkeet Singles',
+		url: 'https://www.teamskeet.com/series/teamskeet-singles',
+		parent: 'teamskeet',
+	},
+	{
+		slug: 'sheseducedme',
+		name: 'She Seduced Me',
+		url: 'https://www.teamskeet.com/series/she-seduced-me',
+		parent: 'teamskeet',
+	},
 	// PERVZ
 	{
 		slug: 'pervzfeatures',
@@ -12789,6 +12899,12 @@ const sites = [
 		url: 'https://www.pervz.com/series/pervdriver',
 		parent: 'pervz',
 	},
+	{
+		slug: 'charmed',
+		name: 'Charmed',
+		url: 'https://www.pervz.com/series/charmed',
+		parent: 'pervz',
+	},
 	// SWAPPZ
 	{
 		slug: 'swappzfeatures',
@@ -12855,6 +12971,13 @@ const sites = [
 		parent: 'freeuse',
 		tags: ['free-use', 'pov'],
 	},
+	{
+		slug: 'freeusesingles',
+		name: 'Freeuse Singles',
+		url: 'https://www.freeuse.com/series/freeuse-singles',
+		parent: 'freeuse',
+		tags: ['free-use'],
+	},
 	// FAMILY STROKES
 	{
 		slug: 'familystrokes',
@@ -12936,6 +13059,14 @@ const sites = [
 		tags: ['family', 'milf'],
 		parent: 'familystrokes',
 	},
+	{
+		slug: 'blackstepdad',
+		name: 'Black StepDad',
+		url: 'https://www.familystrokes.com/series/black-stepdad',
+		alias: ['bsd'],
+		tags: ['family'],
+		parent: 'familystrokes',
+	},
 	// SAY UNCLE
 	{
 		slug: 'blackgodz',
@@ -13528,6 +13659,27 @@ const sites = [
 			},
 		},
 	},
+	// SKM
+	{
+		slug: 'cosplayground',
+		name: 'Cosplayground',
+		url: 'https://cosplayground.com',
+		tags: ['cosplay'],
+		parent: 'ksm',
+		independent: true,
+		parameters: {
+			// seems to use same CMS as Aziani
+			api: 'https://api.cosplayground.com/tour_api.php',
+			seriesId: null,
+			areaId: '2bc2444e-a053-40a2-bc4b-ca3618521563',
+			cdn: 'https://c880f36aa8.mjedge.net',
+			entryIdSlug: true,
+			videos: '/videos',
+			scenePath: '/videos',
+			modelPath: '/models',
+			imageType: 'photos',
+		},
+	},
 	// TEEN CORE CLUB
 	{
 		name: 'Analyzed Girls',

seeds/04_media.js

@@ -706,6 +706,7 @@ const tagMedia = [
 	['blowjob', 'azul_hermosa_realitykings', 'Azul Hermosa and Scott Nails in "Diva For A Day"', 'brazzers'],
 	['blowjob', 3, 'Rose Valie', 'handsonhardcore'],
 	['blowjob', 2, 'Luna Kitsuen in "Gag Reflex"', 'evilangel'],
+	['body-writing', 'emma_rosie_julesjordan', 'Emma Rosie', 'julesjordan'],
 	['bondage', 0, 'Veronica Leal', 'herlimit'],
 	['brunette', 0, 'Darcie Dolce', 'playboy'],
 	['bts', 'charlie_red_private', 'Charlie Red', 'private'],
@@ -984,6 +985,8 @@ const tagMedia = [
 	['mfm', 'hazel_moore_legalporno', 'Hazel Moore', 'analvids'],
 	['mfm', 7, 'Rose Valerie', 'eurosexparties'],
 	['mfm', 6, 'Honey Gold in "Slut Puppies 12"', 'julesjordan'],
+	['threesome', 'dani_daniels_skin_diamond_hollyrandall_2', 'Dani Daniels, Skin Diamond and Erik Everhard in "Threesome Fun"', 'hollyrandall'],
+	// ['threesome', 'dani_daniels_skin_diamond_hollyrandall', 'Dani Daniels, Skin Diamond and Erik Everhard in "Threesome Fun"', 'hollyrandall'],
 	['natural-boobs', 1, 'Nia Nacci', 'firstclasspov'],
 	['natural-boobs', 4, 'Miela (Marry Queen) in "Pure"', 'femjoy'],
 	['natural-boobs', 7, 'Layla London', 'inthecrack'],

seeds/09_manyvids.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const unprint = require('unprint');
+const slugify = require('../src/utils/slugify');
+
+const limit = Number(process.env.MV_LIMIT) || 1000;
+const sorting = process.env.MV_SORT || 'followers'; // followers, top, mostActive, newest, trending
+
+async function fetchCreators(page = 1, acc = []) {
+	console.log(`Fetching ${acc.length}/${limit} creators by ${sorting}`);
+
+	// size seems to be capped at 70
+	const res = await unprint.get(`https://api.manyvids.com/search/creators/list?contentPref=1,2,3&sort=${sorting}&size=50&from=${(page - 1) * 50}`);
+
+	if (res.ok && res.data?.creators) {
+		const creators = acc.concat(res.data.creators.map((creator) => ({
+			name: creator.stageName,
+			slug: `mv.${slugify(creator.slug, '')}`,
+			url: `https://www.manyvids.com/Profile/${creator.id}/${creator.slug}/`,
+			emblem: creator.avatar?.url || null,
+		})));
+
+		if (creators.length >= limit) {
+			return creators;
+		}
+
+		return fetchCreators(page + 1, creators);
+	}
+
+	return acc;
+}
+
+exports.seed = async function (knex) {
+	const channels = await fetchCreators();
+
+	const manyvidsNetwork = await knex('entities')
+		.where('slug', 'manyvids')
+		.where('type', 'network')
+		.first();
+
+	if (!manyvidsNetwork) {
+		throw new Error('ManyVids found, did the network migration run?');
+	}
+
+	const result = await knex('entities')
+		.insert(channels.map((channel) => ({
+			name: channel.name,
+			slug: channel.slug,
+			url: channel.url,
+			parent_id: manyvidsNetwork.id,
+			has_logo: false,
+		})))
+		.onConflict(['slug', 'type'])
+		.merge(['name', 'url'])
+		.returning(knex.raw('(xmax = 0) as inserted'));
+
+	console.log(`Done! ${result.filter((row) => row.inserted).length}/${channels.length} ManyVids creators newly added as channels`);
+};

seeds/ignore/99_fake.js

@@ -1,7 +1,7 @@
 const Promise = require('bluebird');
 const casual = require('casual');
 const fs = require('fs-extra');
-const nanoid = require('nanoid');
+const { nanoid } = require('nanoid');
 const stringify = Promise.promisify(require('csv-stringify'));
 const path = require('path');
 

src/actors.js

@@ -4,11 +4,11 @@ const config = require('config');
 const util = require('util');
 const Promise = require('bluebird');
 const moment = require('moment');
-const blake2 = require('blake2');
+const { blake2b } = require('@noble/hashes/blake2.js');
 const DOMPurify = require('dompurify');
 const { JSDOM } = require('jsdom');
 const omit = require('object.omit');
-const inquirer = require('inquirer');
+const inquirer = require('inquirer').default;
 const unprint = require('unprint');
 
 const { window } = new JSDOM('');
@@ -413,11 +413,7 @@ async function curateProfile(profile, actor) {
 
 		curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
 
-		const hasher = curatedProfile.description && blake2
+		curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(Buffer.from(slugify(curatedProfile.description)), { dkLen: 24 })).toString('hex');
-			.createHash('blake2b', { digestLength: 24 })
-			.update(Buffer.from(slugify(curatedProfile.description)));
-
-		curatedProfile.descriptionHash = curatedProfile.description && hasher.digest('hex');
 
 		curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
 
@@ -611,8 +607,15 @@ async function upsertProfiles(profiles) {
 			media_id: profile.avatarMediaId,
 		}));
 
-		if (avatars.length > 0) {
+		const resolvedAvatars = avatars.filter((avatar) => !!avatar.profile_id);
-			await batchInsert('actors_avatars', avatars, { conflict: false });
+		const unresolvedAvatars = avatars.filter((avatar) => !avatar.profile_id);
+
+		unresolvedAvatars.forEach((avatar) => {
+			logger.warn(`Skipping avatar for actor ${avatar.actor_id} (${avatar.media_id}), no profile could be resolved`);
+		});
+
+		if (resolvedAvatars.length > 0) {
+			await batchInsert('actors_avatars', resolvedAvatars, { conflict: false });
 		}
 	}
 }
@@ -981,6 +984,7 @@ async function getOrCreateActors(baseActors, batchId) {
 				...actor,
 				actorId,
 				update: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
+				profileId: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
 			};
 		})
 		.filter((actor) => !!actor.actorId)

src/alerts.js

@@ -1,283 +0,0 @@
-'use strict';
-
-const escapeRegexp = require('escape-string-regexp');
-
-const argv = require('./argv');
-const logger = require('./logger')(__filename);
-const knex = require('./knex');
-const { indexApi } = require('./manticore');
-const bulkInsert = require('./utils/bulk-insert');
-const { HttpError } = require('./errors');
-
-async function addAlert(alert, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	if ((!alert.actors || alert.actors.length === 0) && (!alert.tags || alert.tags.length === 0) && (!alert.entities || alert.entities.length === 0) && (!alert.matches || alert.matches.length === 0)) {
-		throw new HttpError('Alert must contain at least one actor, tag or entity', 400);
-	}
-
-	if (alert.matches?.some((match) => !match.property || !match.expression)) {
-		throw new HttpError('Match must define a property and an expression', 400);
-	}
-
-	const [{ id: alertId }] = await knex('alerts')
-		.insert({
-			user_id: sessionUser.id,
-			notify: alert.notify,
-			email: alert.email,
-			all: alert.all,
-		})
-		.returning('id');
-
-	await Promise.all([
-		alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map((actorId) => ({
-			alert_id: alertId,
-			actor_id: actorId,
-		})), false),
-		alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map((tagId) => ({
-			alert_id: alertId,
-			tag_id: tagId,
-		})), false),
-		alert.matches?.length > 0 && bulkInsert('alerts_matches', alert.matches.map((match) => ({
-			alert_id: alertId,
-			property: match.property,
-			expression: /\/.*\//.test(match.expression)
-				? match.expression.slice(1, -1)
-				: escapeRegexp(match.expression),
-		})), false),
-		alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
-			alert_id: alertId,
-			stash_id: stashId,
-		})), false),
-		alert.entities && bulkInsert('alerts_entities', alert.entities.map((entityId) => ({
-			alert_id: alertId,
-			entity_id: entityId,
-		})).slice(0, alert.all ? 1 : Infinity), false), // one scene can never match multiple entities in AND mode
-	]);
-
-	return alertId;
-}
-
-async function removeAlert(alertId) {
-	await knex('alerts').where('id', alertId).delete();
-}
-
-async function notify(scenes) {
-	if (argv.showcased === false) {
-		logger.info(`Skipping notify for ${scenes.length} scenes because showcase flag is disabled.`);
-		return [];
-	}
-
-	const sceneIds = scenes.map((scene) => scene.id);
-
-	const [
-		releasesActors,
-		releasesTags,
-		rawAlerts,
-		alertsActors,
-		alertsTags,
-		alertsEntities,
-		alertsMatches,
-		alertsStashes,
-	] = await Promise.all([
-		knex('releases_actors').whereIn('release_id', sceneIds),
-		knex('releases_tags').whereIn('release_id', sceneIds),
-		knex('alerts'),
-		knex('alerts_actors'),
-		knex('alerts_tags'),
-		knex('alerts_entities'),
-		knex('alerts_matches'),
-		knex('alerts_stashes'),
-	]);
-
-	const actorIdsByReleaseId = releasesActors.reduce((acc, releaseActor) => {
-		if (!acc[releaseActor.release_id]) {
-			acc[releaseActor.release_id] = [];
-		}
-
-		acc[releaseActor.release_id].push(releaseActor.actor_id);
-
-		return acc;
-	}, {});
-
-	const tagIdsByReleaseId = releasesTags.reduce((acc, releaseTag) => {
-		if (!acc[releaseTag.release_id]) {
-			acc[releaseTag.release_id] = [];
-		}
-
-		acc[releaseTag.release_id].push(releaseTag.tag_id);
-
-		return acc;
-	}, {});
-
-	const alertsActorsByAlertId = alertsActors.reduce((acc, alertActor) => { if (!acc[alertActor.alert_id]) { acc[alertActor.alert_id] = []; } acc[alertActor.alert_id].push(alertActor.actor_id); return acc; }, {});
-	const alertsTagsByAlertId = alertsTags.reduce((acc, alertTag) => { if (!acc[alertTag.alert_id]) { acc[alertTag.alert_id] = []; } acc[alertTag.alert_id].push(alertTag.tag_id); return acc; }, {});
-	const alertsEntitiesByAlertId = alertsEntities.reduce((acc, alertEntity) => { if (!acc[alertEntity.alert_id]) { acc[alertEntity.alert_id] = []; } acc[alertEntity.alert_id].push(alertEntity.entity_id); return acc; }, {});
-	const alertsStashesByAlertId = alertsStashes.reduce((acc, alertStash) => { if (!acc[alertStash.alert_id]) { acc[alertStash.alert_id] = []; } acc[alertStash.alert_id].push(alertStash.stash_id); return acc; }, {});
-
-	const alertsMatchesByAlertId = alertsMatches.reduce((acc, alertMatch) => {
-		if (!acc[alertMatch.alert_id]) {
-			acc[alertMatch.alert_id] = [];
-		}
-
-		acc[alertMatch.alert_id].push({
-			property: alertMatch.property,
-			expression: new RegExp(alertMatch.expression, 'ui'),
-		});
-
-		return acc;
-	}, {});
-
-	const alerts = rawAlerts.map((alert) => ({
-		id: alert.id,
-		userId: alert.user_id,
-		notify: alert.notify,
-		email: alert.email,
-		all: alert.all,
-		allActors: alert.all_actors,
-		allEntities: alert.all_entities,
-		allTags: alert.all_tags,
-		allMatches: alert.all_matches,
-		actors: alertsActorsByAlertId[alert.id] || [],
-		tags: alertsTagsByAlertId[alert.id] || [],
-		entities: alertsEntitiesByAlertId[alert.id] || [],
-		matches: alertsMatchesByAlertId[alert.id] || [],
-		stashes: alertsStashesByAlertId[alert.id] || [],
-	}));
-
-	const curatedScenes = scenes.map((scene) => ({
-		id: scene.id,
-		title: scene.title,
-		description: scene.description,
-		actorIds: actorIdsByReleaseId[scene.id] || [],
-		tagIds: tagIdsByReleaseId[scene.id] || [],
-		entityId: scene.entity.id,
-		parentEntityId: scene.entity.parent?.id,
-	}));
-
-	const triggers = alerts.flatMap((alert) => {
-		const alertScenes = curatedScenes.filter((scene) => {
-			if (alert.all) {
-				if (alert.actors.length > 0 && !alert.actors[alert.allActors ? 'every' : 'some']((actorId) => scene.actorIds.includes(actorId))) {
-					return false;
-				}
-
-				if (alert.tags.length > 0 && !alert.tags[alert.allTags ? 'every' : 'some']((tagId) => scene.tagIds.includes(tagId))) {
-					return false;
-				}
-
-				// multiple entities can only be matched in OR mode
-				if (alert.entities.length > 0 && !alert.entities.some((alertEntityId) => alertEntityId === scene.entityId || alertEntityId === scene.parentEntityId)) {
-					return false;
-				}
-
-				if (alert.matches.length > 0 && !alert.matches[alert.allMatches ? 'every' : 'some']((match) => match.expression.test(scene[match.property]))) {
-					return false;
-				}
-
-				return true;
-			}
-
-			if (alert.matches.length > 0 && alert.actors[alert.allActors ? 'every' : 'some']((actorId) => scene.actorIds.includes(actorId))) {
-				return true;
-			}
-
-			if (alert.tags.length > 0 && alert.tags[alert.allTags ? 'every' : 'some']((tagId) => scene.tagIds.includes(tagId))) {
-				return true;
-			}
-
-			// multiple entities can only be matched in OR mode
-			if (alert.entities.length > 0 && alert.entities.some((alertEntityId) => alertEntityId === scene.entityId || alertEntityId === scene.parentEntityId)) {
-				return true;
-			}
-
-			if (alert.matches.length > 0 && alert.matches[alert.allMatches ? 'every' : 'some']((match) => match.expression.test(scene[match.property]))) {
-				return true;
-			}
-
-			return false;
-		});
-
-		return alertScenes.map((scene) => ({
-			sceneId: scene.id,
-			alert,
-		}));
-	});
-
-	const notifications = Object.values(Object.fromEntries(triggers // prevent multiple notifications for the same scene
-		.filter((trigger) => trigger.alert.notify)
-		.map((trigger) => [`${trigger.alert.userId}:${trigger.sceneId}`, trigger])))
-		.map((trigger) => ({
-			user_id: trigger.alert.userId,
-			alert_id: trigger.alert.id,
-			scene_id: trigger.sceneId,
-		}));
-
-	const uniqueStashes = Object.values(Object.fromEntries(triggers.flatMap((trigger) => trigger.alert.stashes.map((stashId) => ({
-		stashId,
-		sceneId: trigger.sceneId,
-		userId: trigger.alert.userId,
-	}))).map((stash) => [`${stash.stashId}:${stash.sceneId}`, stash])));
-
-	const stashEntries = uniqueStashes.map((stash) => ({
-		scene_id: stash.sceneId,
-		stash_id: stash.stashId,
-	}));
-
-	const [stashed] = await Promise.all([
-		bulkInsert('stashes_scenes', stashEntries, false),
-		bulkInsert('notifications', notifications, false),
-	]);
-
-	// we need created_at from the databased, but user_id is not returned. it's easier to query it than to try and merge it with the input data
-	const stashedEntries = await knex('stashes_scenes')
-		.select('stashes_scenes.*', 'stashes.user_id')
-		.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
-		.whereIn('stashes_scenes.id', stashed.map((stash) => stash.id));
-
-	const docs = stashedEntries.map((stash) => ({
-		replace: {
-			index: 'scenes_stashed',
-			id: stash.id,
-			doc: {
-				scene_id: stash.scene_id,
-				user_id: stash.user_id,
-				stash_id: stash.stash_id,
-				created_at: Math.round(stash.created_at.getTime() / 1000),
-			},
-		},
-	}));
-
-	if (docs.length > 0) {
-		await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-	}
-
-	return triggers;
-}
-
-async function updateNotification(notificationId, notification, sessionUser) {
-	await knex('notifications')
-		.where('user_id', sessionUser.id)
-		.where('id', notificationId)
-		.update({
-			seen: notification.seen,
-		});
-}
-
-async function updateNotifications(notification, sessionUser) {
-	await knex('notifications')
-		.where('user_id', sessionUser.id)
-		.update({
-			seen: notification.seen,
-		});
-}
-
-module.exports = {
-	addAlert,
-	removeAlert,
-	notify,
-	updateNotification,
-	updateNotifications,
-};

src/app.js

@@ -5,13 +5,12 @@ const startTime = new Date();
 const config = require('config');
 const util = require('util');
 const unprint = require('unprint');
-const log = require('why-is-node-running');
+const log = require('why-is-node-running').default;
 const Inspector = require('inspector-api');
 const fs = require('fs').promises;
 const { format, intervalToDuration } = require('date-fns');
 
 const argv = require('./argv');
-const http = require('./utils/http');
 
 const logger = require('./logger')(__filename);
 const knex = require('./knex');
@@ -34,7 +33,7 @@ const unprintOptions = {
 	logErrors: false,
 	timeout: argv.requestTimeout,
 	userAgent: 'traxxx',
-	browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+	// browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
 	apiUserAgent: 'traxxx',
 	clientRetirement: config.bypass.browser.clientRetirement,
 	remote: config.bypass.browser.remote,
@@ -210,7 +209,7 @@ async function init() {
 		}
 
 		if (argv.request) {
-			const res = await http[argv.requestMethod](argv.request);
+			const res = await unprint.request(argv.request, { method: argv.requestMethod, interface: 'request' });
 
 			console.log(res.status, res.body);
 		}
@@ -238,7 +237,7 @@ async function init() {
 		const movieUrls = (argv.movie || []).concat(moviesFromFile || []);
 
 		const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
-		const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;
+		const deepMovies = movieUrls.length > 0 || sceneMovies.length > 0 ? await fetchMovies([...movieUrls, ...sceneMovies]) : sceneMovies;
 
 		const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
 		const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
@@ -252,7 +251,18 @@ async function init() {
 			const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
 			const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
 
-			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
+			const allScenes = [...storedScenes, ...storedMovieScenes];
+
+			await associateMovieScenes(storedMovies, allScenes);
+
+			// refreshing the materialized views is slow, skip when nothing was stored (e.g. --request)
+			if (allScenes.length > 0) {
+				await updateSceneSearch(allScenes.map((scene) => scene.id));
+			}
+
+			if (storedMovies.length > 0) {
+				await updateMovieSearch(storedMovies.map((movie) => movie.id));
+			}
 		}
 
 		logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() }))
@@ -264,12 +274,11 @@ async function init() {
 		logger.error(error);
 	}
 
-	await http.destroyBypassSessions();
-	await http.destroyBrowserSessions();
 	await unprint.closeAllBrowsers();
+	await unprint.closeAgents();
 
 	knex.destroy();
-	redis.disconnect();
+	redis.destroy();
 
 	done = true;
 }

src/argv.js

@@ -2,6 +2,7 @@
 
 const config = require('config');
 const yargs = require('yargs');
+const { hideBin } = require('yargs/helpers');
 const moment = require('moment');
 
 function interpretDate(after, ignoreIfEmpty = false) {
@@ -27,7 +28,7 @@ function interpretDate(after, ignoreIfEmpty = false) {
 		.toDate();
 }
 
-const { argv } = yargs
+const { argv } = yargs(hideBin(process.argv))
 	.command('npm start')
 	.command('update', 'fetch latest updates', () => {
 		console.log('UPDATING!');

src/auth.js

@@ -1,110 +0,0 @@
-'use strict';
-
-const config = require('config');
-const util = require('util');
-const crypto = require('crypto');
-
-const knex = require('./knex');
-const { curateUser, fetchUser } = require('./users');
-const { HttpError } = require('./errors');
-
-const scrypt = util.promisify(crypto.scrypt);
-
-async function verifyPassword(password, storedPassword) {
-	const [salt, hash] = storedPassword.split('/');
-	const hashedPassword = (await scrypt(password, salt, 64)).toString('hex');
-
-	if (hashedPassword === hash) {
-		return true;
-	}
-
-	throw new HttpError('Username or password incorrect', 401);
-}
-
-async function login(credentials) {
-	if (!config.auth.login) {
-		throw new HttpError('Authentication is disabled', 405);
-	}
-
-	const user = await fetchUser(credentials.username.trim(), true);
-
-	if (!user) {
-		throw new HttpError('Username or password incorrect', 401);
-	}
-
-	await verifyPassword(credentials.password, user.password);
-
-	await knex('users')
-		.update('last_login', 'NOW()')
-		.where('id', user.id);
-
-	return curateUser(user);
-}
-
-async function signup(credentials) {
-	if (!config.auth.signup) {
-		throw new HttpError('Authentication is disabled', 405);
-	}
-
-	const curatedUsername = credentials.username.trim();
-
-	if (!curatedUsername) {
-		throw new HttpError('Username required', 400);
-	}
-
-	if (curatedUsername.length < config.auth.usernameLength[0]) {
-		throw new HttpError('Username is too short', 400);
-	}
-
-	if (curatedUsername.length > config.auth.usernameLength[1]) {
-		throw new HttpError('Username is too long', 400);
-	}
-
-	if (!config.auth.usernamePattern.test(curatedUsername)) {
-		throw new HttpError('Username contains invalid characters', 400);
-	}
-
-	if (!credentials.email) {
-		throw new HttpError('E-mail required', 400);
-	}
-
-	if (credentials.password?.length < 3) {
-		throw new HttpError('Password must be 3 characters or longer', 400);
-	}
-
-	const existingUser = await knex('users')
-		.where('username', curatedUsername)
-		.orWhere('email', credentials.email)
-		.first();
-
-	if (existingUser) {
-		throw new HttpError('Username or e-mail already in use', 409);
-	}
-
-	const salt = crypto.randomBytes(16).toString('hex');
-	const hashedPassword = (await scrypt(credentials.password, salt, 64)).toString('hex');
-	const storedPassword = `${salt}/${hashedPassword}`;
-
-	const [{ id: userId }] = await knex('users')
-		.insert({
-			username: curatedUsername,
-			email: credentials.email,
-			password: storedPassword,
-		})
-		.returning('id');
-
-	await knex('stashes').insert({
-		user_id: userId,
-		name: 'Favorites',
-		slug: 'favorites',
-		public: false,
-		primary: true,
-	});
-
-	return fetchUser(userId);
-}
-
-module.exports = {
-	login,
-	signup,
-};

src/biometrics.js

@@ -3,7 +3,13 @@
 const config = require('config');
 const path = require('path');
 const fsPromises = require('fs').promises;
-const Human = require('@vladmandic/human').default;
+const util = require('util');
+
+// tfjs-node still uses util helpers that were removed (isNullOrUndefined) or deprecated with a runtime warning (isArray) in Node 23+, must be set before it loads
+util.isNullOrUndefined ??= (value) => value === null || value === undefined;
+util.isArray = Array.isArray;
+
+const Human = require('@vladmandic/human').default; // eslint-disable-line import/order
 
 const knex = require('./knex');
 const logger = require('./logger')(__filename);

src/deep.js

@@ -9,9 +9,7 @@ const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
 const { fetchReleaseEntities, urlToHostname } = require('./entities');
 const logger = require('./logger')(__filename);
-const qu = require('./utils/qu');
 const getRecursiveParameters = require('./utils/get-recursive-parameters');
-const windows = require('./utils/http-windows');
 
 const waitImmediate = util.promisify(setImmediate);
 
@@ -85,35 +83,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
 	}
 
 	if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
-		/*
+		return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
-			return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		}
-		*/
-
-		if (!scraper.deprecated) {
-			return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
-		}
-
-		const session = qu.session();
-
-		const res = await qu.get(url, null, null, {
-			session,
-			rejectUnauthorized: false,
-		});
-
-		const cookie = await session._sessionOptions.cookieJar.get(url);
-
-		if (res.ok) {
-			return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
-				session,
-				headers: res.headers,
-				cookieJar: session._sessionOptions.cookieJar,
-				cookie,
-			});
-		}
-
-		return res.status;
 	}
 
 	return null;
@@ -123,15 +93,21 @@ function fetchMovie(scraper, url, entity, baseRelease, options) {
 	return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
 }
 
-async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
+function shouldDeepScrape(baseRelease) {
-	const entity = baseRelease.entity || entitiesByHostname[urlToHostname(baseRelease.url)];
+	return argv.deep && !!(baseRelease.url || baseRelease.path || baseRelease.forceDeep);
+}
+
+async function scrapeRelease(baseRelease, entities, type = 'scene') {
+	const entity = entities.byId[baseRelease.entity?.id]
+		|| entities.byHostname[urlToHostname(baseRelease.url)]
+		|| baseRelease.entity;
 
 	if (!entity) {
 		logger.warn(`No entity available for ${baseRelease.url}`);
 		return baseRelease;
 	}
 
-	if ((!baseRelease.url && !baseRelease.path && !baseRelease.forceDeep) || !argv.deep) {
+	if (!shouldDeepScrape(baseRelease)) {
 		return {
 			...baseRelease,
 			entity,
@@ -165,19 +141,10 @@ async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
 			? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options)
 			: await fetchMovie(layoutScraper, baseRelease.url, entity, baseRelease, options);
 
-		const pathname = baseRelease.path || (baseRelease.url && new URL(baseRelease.url).pathname.replace(/\//g, '_'));
-
 		if (rawScrapedRelease) {
 			delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
 		}
 
-		if (windows.has(pathname)) {
-			logger.debug(`Closing window for ${pathname}`);
-
-			windows.get(pathname).close();
-			windows.delete(pathname);
-		}
-
 		await waitImmediate;
 
 		// logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
@@ -247,22 +214,37 @@ async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
 }
 
 async function scrapeReleases(baseReleases, entitiesByHostname, type) {
-	const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesByHostname).map(async ([slug, entity]) => {
+	// entities attached to base releases (e.g. from updates) are not included in entitiesByHostname
-		if (entity.scraper?.beforeFetchScenes) {
+	const entitiesById = Object.fromEntries([
+		...Object.values(entitiesByHostname),
+		...baseReleases.map((baseRelease) => baseRelease.entity),
+	].filter(Boolean).map((entity) => [entity.id, entity]));
+
+	// only run before hook for entities that will actually be deep scraped
+	const deepEntityIds = new Set(baseReleases
+		.filter((baseRelease) => shouldDeepScrape(baseRelease))
+		.map((baseRelease) => baseRelease.entity?.id || entitiesByHostname[urlToHostname(baseRelease.url)]?.id)
+		.filter(Boolean));
+
+	const entitiesWithBeforeDataById = Object.fromEntries(await Promise.all(Object.values(entitiesById).map(async (entity) => {
+		if (deepEntityIds.has(entity.id) && entity.scraper?.beforeFetchScenes) {
 			const parameters = getRecursiveParameters(entity);
 			const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
 
-			return [slug, { ...entity, preData }];
+			return [entity.id, { ...entity, preData }];
 		}
 
-		return [slug, entity];
+		return [entity.id, entity];
-	}));
+	})));
 
-	const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean));
+	const entitiesWithBeforeData = {
+		byId: entitiesWithBeforeDataById,
+		byHostname: Object.fromEntries(Object.entries(entitiesByHostname).map(([hostname, entity]) => [hostname, entitiesWithBeforeDataById[entity.id]])),
+	};
 
 	return Promise.map(
 		baseReleases,
-		async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
+		async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeData, type),
 		{ concurrency: 1 },
 	);
 }

src/entities.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const config = require('config');
-const inquirer = require('inquirer');
+const inquirer = require('inquirer').default;
 
 const logger = require('./logger')(__filename);
 const argv = require('./argv');

src/init.js

@@ -1,4 +1,3 @@
-require('babel-polyfill');
 const init = require('./app');
 
 init();

src/manticore.js

@@ -1,12 +0,0 @@
-const config = require('config');
-const manticore = require('manticoresearch');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-module.exports = {
-	searchApi: new manticore.SearchApi(mantiClient),
-	indexApi: new manticore.IndexApi(mantiClient),
-	utilsApi: new manticore.UtilsApi(),
-};

src/media.js

@@ -8,13 +8,14 @@ const fsPromises = require('fs').promises;
 const path = require('path');
 const stream = require('stream');
 const { nanoid } = require('nanoid/non-secure');
-const mime = require('mime');
+const mime = require('mime').default;
 // const fileType = require('file-type');
 const ffmpeg = require('fluent-ffmpeg');
 const sharp = require('sharp');
-const blake2 = require('blake2');
+const { blake2b } = require('@noble/hashes/blake2.js');
 const { format } = require('date-fns');
 const taskQueue = require('promise-task-queue');
+const unprint = require('unprint');
 
 const { Upload } = require('@aws-sdk/lib-storage');
 const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
@@ -22,14 +23,15 @@ const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
 const logger = require('./logger')(__filename);
 const argv = require('./argv');
 const knex = require('./knex');
-const http = require('./utils/http');
 const batchInsert = require('./utils/batch-insert');
 const chunk = require('./utils/chunk');
-const { get } = require('./utils/qu');
 const { fetchEntityReleaseIds } = require('./entity-releases');
 
 const streamQueue = taskQueue();
 
+// carried over from the legacy HTTP module, unprint's raw request interface would identify as traxxx
+const mediaUserAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36';
+
 const s3 = new S3Client({
 	region: 'eu-central-1',
 	endpoint: 'https://s3.eu-central-1.wasabisys.com',
@@ -383,10 +385,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
 		};
 	}
 
-	const res = await get(baseSource.url);
+	const res = await unprint.get(baseSource.url);
 
 	if (res.ok) {
-		const src = await baseSource.extract(res.item);
+		const src = await baseSource.extract(res.context);
 
 		return {
 			...baseSource,
@@ -637,24 +639,28 @@ async function storeFile(media, options) {
 }
 
 async function fetchHttpSource(source, tempFileTarget, hashStream) {
-	const res = await http.get(source.src, {
+	const res = await unprint.get(source.src, {
-		limits: 'media',
+		interface: 'request',
+		limiter: 'media',
 		headers: {
+			'user-agent': mediaUserAgent,
 			// explicit host not allowed in HTTP/2
 			// host: new URL(source.src).hostname,
 			// ...(source.host && { host: source.host }),
 			...(source.referer && { referer: source.referer }),
 		},
-		stream: true, // sources are fetched in parallel, don't gobble up memory
+		tls: { rejectUnauthorized: false },
-		followRedirects: source.followRedirects,
+		// sources are fetched in parallel, stream to file instead of gobbling up memory
 		transforms: [hashStream],
 		destination: tempFileTarget,
-		...(source.interval && { interval: source.interval }),
+		// undefined would override the defaults
-		...(source.concurrency && { concurrency: source.concurrency }),
+		...(typeof source.followRedirects === 'boolean' && { followRedirects: source.followRedirects }),
+		...((argv.mediaInterval ?? source.interval) && { interval: argv.mediaInterval ?? source.interval }),
+		...((argv.mediaConcurrency ?? source.concurrency) && { concurrency: argv.mediaConcurrency ?? source.concurrency }),
 	});
 
 	if (!res.ok) {
-		throw new Error(`Response ${res.status} not OK`);
+		throw new Error(`Response ${res.status} not OK${res.statusText ? ` (${res.statusText})` : ''}`); // status 200 if stream was interrupted
 	}
 
 	return {
@@ -723,7 +729,7 @@ async function fetchSource(source, baseMedia) {
 		const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
 		let tempFileTarget;
 
-		const hasher = blake2.createHash('blake2b', { digestLength: 24 });
+		const hasher = blake2b.create({ dkLen: 24 });
 
 		let size = 0;
 
@@ -743,7 +749,7 @@ async function fetchSource(source, baseMedia) {
 				? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
 				: await fetchHttpSource(source, tempFileTarget, hashStream);
 
-			const hash = hasher.digest('hex');
+			const hash = Buffer.from(hasher.digest()).toString('hex');
 
 			const [type, subtype] = mimetype.split('/');
 			const extension = mime.getExtension(mimetype);
@@ -772,6 +778,8 @@ async function fetchSource(source, baseMedia) {
 			hashStream.destroy();
 			tempFileTarget?.destroy();
 
+			await fsPromises.unlink(tempFilePath).catch(() => { /* probably never created */ });
+
 			if (error.code !== 'VERIFY_TYPE') {
 				logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
 
@@ -914,6 +922,9 @@ async function storeMedias(baseMedias, options) {
 			async (baseMedia) => storeFile(baseMedia, options),
 			{ concurrency: 100 }, // don't overload disk
 		);
+	} else {
+		// duplicates aren't stored, clean up their fetched temp files
+		await Promise.all(existingHashMedias.map(async (media) => media.file?.path && fsPromises.unlink(media.file.path).catch(() => { /* probably already removed */ })));
 	}
 
 	const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));

src/releases.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const inquirer = require('inquirer');
+const inquirer = require('inquirer').default;
 
 const logger = require('./logger')(__filename);
 const knex = require('./knex');

src/scrapers/actors.js

@@ -29,7 +29,6 @@ const karups = require('./karups');
 const kellymadison = require('./kellymadison');
 const kink = require('./kink');
 const mariskax = require('./mariskax');
-// const analvids = require('./analvids');
 const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
 const testedefudelidade = require('./testedefudelidade');
 const theflourish = require('./theflourish');
 const tokyohot = require('./tokyohot');
-// const topwebmodels = require('./topwebmodels');
 const virtualtaboo = require('./virtualtaboo');
 const vixen = require('./vixen');
 const wankzvr = require('./wankzvr');
@@ -229,7 +227,6 @@ module.exports = {
 	boyfun: karups,
 	kellymadison,
 	'8kmembers': kellymadison,
-	// analvids,
 	analvids: pornbox,
 	littlecapricedreams,
 	mariskax,
@@ -279,6 +276,7 @@ module.exports = {
 	// etc
 	'18vr': badoink,
 	theflourishxxx: theflourish,
+	cosplayground: aziani,
 	pierrewoodman,
 	exploitedx, // only from known URL that will specify site
 	fullpornnetwork,

src/scrapers/adultempire.js

@@ -62,7 +62,7 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
 		};
 	});
 
-	release.tags = query.contents('.tags a, .categories a');
+	release.tags = query.contents('.tags a, .categories a, .tags + div a[data-label="Attribute"]');
 	release.studio = parameters?.studio === false ? null : slugify(query.content('.studio span:last-child, .studio a'), '');
 
 	if (type === 'scene') {
@@ -103,6 +103,18 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
 		release.qualities = [2160];
 	}
 
+	const serieTitle = query.content('.series a');
+	const serieUrl = query.url('.series a');
+	const serieId = serieUrl && new URL(serieUrl).searchParams.get('series');
+
+	if (serieTitle && serieId) {
+		release.serie = {
+			entryId: serieId,
+			title: serieTitle,
+			url: serieUrl,
+		};
+	}
+
 	return release;
 }
 

src/scrapers/americanpornstar.js

@@ -1,45 +1,45 @@
 'use strict';
 
-const { fetchLatest, fetchProfile } = require('./julesjordan');
+const unprint = require('unprint');
 
-const qu = require('../utils/q');
+const { fetchLatest, fetchProfile } = require('./julesjordan');
 const slugify = require('../utils/slugify');
 
 function scrapeScene({ query }, channel) {
 	const release = {};
-	const title = query.cnt('.update_title', false);
+	const title = query.content('.update_title', { trim: false });
 
 	release.title = title.trim();
-	release.description = query.cnt('.latest_update_description');
+	release.description = query.content('.latest_update_description');
 
 	release.date = query.date('.update_date', 'MM/DD/YYYY');
 	release.actors = query.all('.tour_update_models a').map((actorEl) => ({
-		name: query.cnt(actorEl),
+		name: unprint.query.content(actorEl),
-		url: query.url(actorEl, null),
+		url: unprint.query.url(actorEl, null),
 	}));
 
 	release.entryId = slugify(release.title);
 
-	release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
+	release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
 
-	const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
+	const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
-	const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
+	const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
 
 	release.poster = [
 		poster.replace(/.jpg$/, '-full.jpg'),
 		poster,
 	];
 
-	release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
+	release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
 		img.replace(/.jpg$/, '-full.jpg'),
 		img,
 	]);
 
 	if (trailer) {
-		release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
+		release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
 	}
 
-	release.tags = query.cnts('.tour_update_tags a');
+	release.tags = query.contents('.tour_update_tags a');
 
 	return release;
 }
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel);
+		return scrapeScene(res.context, channel);
 	}
 
 	return res.status;

src/scrapers/analvids.js

@@ -1,174 +0,0 @@
-'use strict';
-
-const unprint = require('unprint');
-
-const slugify = require('../utils/slugify');
-
-function extractTitle(originalTitle) {
-	const titleComponents = originalTitle.split(' ');
-	// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
-	const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
-	const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
-	const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
-
-	return { shootId, title };
-}
-
-function scrapeAll(scenes, channel) {
-	return scenes.map(({ query }) => {
-		const release = {};
-
-		release.url = query.url('.card-scene__view > a');
-		release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
-
-		release.title = query.content('.card-scene__text');
-		release.shootId = extractTitle(release.title).shootId;
-
-		release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
-		release.duration = query.duration('.label--time:first-child');
-
-		release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
-		const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
-
-		if (caps && release.poster) {
-			release.caps = caps;
-		} else if (caps) {
-			[release.poster, ...release.caps] = caps;
-		}
-
-		release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
-
-		return release;
-	});
-}
-
-async function fetchLatest(channel, page) {
-	// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
-	// studios as channels
-	const url = `${channel.url}/latest/${page}`;
-
-	const res = await unprint.get(url, {
-		selectAll: '.card-scene',
-		headers: {
-			Referer: url,
-		},
-	});
-
-	if (res.ok) {
-		return scrapeAll(res.context, channel);
-	}
-
-	return res.status;
-}
-
-function scrapeScene({ query }, url) {
-	const release = {};
-
-	release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
-
-	const featuring = query.content('.watch__title .watch__featuring_models');
-
-	release.title = query.content('.watch__title').replace(featuring, '');
-	release.description = query.content('.text-mob-more');
-	release.shootId = extractTitle(release.title).shootId;
-
-	release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
-	release.duration = query.duration('.bi-clock');
-
-	release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
-		name: unprint.query.content(el),
-		url: unprint.query.url(el, null),
-	}));
-
-	release.tags = query.contents('.genres-list a[href*="/genre"]');
-
-	// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
-
-	release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
-
-	release.trailer = query.all('.watch__video source').map((el) => ({
-		src: unprint.query.video(el, null, { attribute: 'src' }),
-		quality: unprint.query.number(el, null, { attribute: 'size' }),
-	}));
-
-	return release;
-}
-
-async function fetchScene(url) {
-	const res = await unprint.get(url, {
-		headers: {
-			Referer: url,
-		},
-	});
-
-	if (res.ok) {
-		return scrapeScene(res.context, url);
-	}
-
-	return res.status;
-}
-
-function scrapeProfile({ query }, url, channel) {
-	const profile = { url };
-
-	profile.nationality = query.content('.model__info a[href*="/nationality"]');
-	profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
-
-	profile.avatar = query.img('.model__left img');
-
-	profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
-
-	return profile;
-}
-
-async function getActorUrl(actor, channel) {
-	if (actor.url) {
-		return actor.url;
-	}
-
-	const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
-
-	const searchRes = await unprint.get(searchUrl, {
-		headers: {
-			Referer: actor.url,
-		},
-	});
-
-	if (!searchRes.ok) {
-		return searchRes.status;
-	}
-
-	const result = searchRes.body.terms.find((item) => item.type === 'model');
-
-	if (result) {
-		return result.url;
-	}
-
-	return null;
-}
-
-async function fetchProfile(actor, { channel }) {
-	const actorUrl = await getActorUrl(actor, channel);
-
-	if (typeof actorUrl !== 'string') {
-		return actorUrl;
-	}
-
-	const bioRes = await unprint.get(actorUrl, {
-		headers: {
-			Referer: actor.url,
-		},
-	});
-
-	if (bioRes.ok) {
-		return scrapeProfile(bioRes.context, actorUrl, channel);
-	}
-
-	return bioRes.status;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchScene,
-	fetchProfile,
-};

src/scrapers/analvids_legacy.js

@@ -1,203 +0,0 @@
-'use strict';
-
-const { JSDOM } = require('jsdom');
-const cheerio = require('cheerio');
-const moment = require('moment');
-
-const http = require('../utils/http');
-const slugify = require('../utils/slugify');
-
-function extractTitle(originalTitle) {
-	const titleComponents = originalTitle.split(' ');
-	// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
-	const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
-	const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
-	const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
-
-	return { shootId, title };
-}
-
-function getPoster(posterElement, sceneId) {
-	const posterStyle = posterElement.attr('style');
-
-	if (posterStyle) {
-		return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
-	}
-
-	const posterRange = posterElement.attr('data-casting');
-	const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
-	const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
-
-	if (!posterTimeRange) {
-		return null;
-	}
-
-	if (typeof posterTimeRange === 'number') {
-		// poster time is already a single time value
-		return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
-	}
-
-	const [max, min] = posterTimeRange.split('-');
-	const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
-
-	return `https://analvids.com/casting/${sceneId}/${posterTime}`;
-}
-
-function scrapeAll(html) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
-	const scenesElements = $('.thumbnails > div').toArray();
-
-	return scenesElements.map((element) => {
-		const sceneLinkElement = $(element).find('.thumbnail-title a');
-		const url = sceneLinkElement.attr('href');
-
-		const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
-		const { shootId, title } = extractTitle(originalTitle);
-		const entryId = new URL(url).pathname.split('/')[2];
-
-		const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
-
-		const sceneId = $(element).attr('data-content');
-		const posterElement = $(element).find('.thumbnail-avatar');
-
-		const poster = getPoster(posterElement, sceneId);
-
-		return {
-			url,
-			shootId,
-			entryId,
-			title,
-			date,
-			poster,
-		};
-	});
-}
-
-async function scrapeScene(html, url, site, useGallery) {
-	const $ = cheerio.load(html, { normalizeWhitespace: true });
-	const playerObject = $('script:contains("new WatchPage")').html();
-	const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
-	const data = playerData && JSON.parse(playerData);
-
-	const release = { url };
-
-	const originalTitle = $('h1.watchpage-title').text().trim();
-	const { shootId, title } = extractTitle(originalTitle);
-
-	release.shootId = shootId;
-	release.entryId = new URL(url).pathname.split('/')[2];
-
-	release.title = title;
-	release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
-
-	const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
-
-	release.description = $('meta[name="description"]')?.attr('content')?.trim()
-        || (descriptionElement && $(descriptionElement).find('dd').text().trim());
-
-	release.actors = $(actorsElement)
-		.find('a[href*="com/model"]')
-		.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
-
-	release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
-	release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-
-	const photos = useGallery
-		? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
-		: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
-
-	release.photos = photos.map((source) => {
-		// source without parameters sometimes serves larger preview photo
-		const { origin, pathname } = new URL(source);
-
-		return `${origin}${pathname}`;
-	});
-
-	const posterStyle = $('#player').attr('style');
-	const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
-
-	release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
-
-	if (data) {
-		const qualityMap = {
-			web: 240,
-			vga: 480,
-			hd: 720,
-			'1080p': 1080,
-		};
-
-		release.trailer = data.clip.qualities.map((trailer) => ({
-			src: trailer.src,
-			type: trailer.type,
-			quality: qualityMap[trailer.quality] || trailer.quality,
-		}));
-	}
-
-	const studioName = $('.watchpage-studioname').first().text().trim();
-	release.studio = slugify(studioName, '', { removePunctuation: true });
-
-	return release;
-}
-
-async function scrapeProfile(html, _url, actorName) {
-	const { document } = new JSDOM(html).window;
-
-	const profile = {
-		name: actorName,
-	};
-
-	const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
-	const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
-
-	const bio = entries
-		.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
-		.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
-
-	profile.birthPlace = bio.Nationality;
-
-	if (bio.Age) profile.age = bio.Age;
-	if (avatarEl) profile.avatar = avatarEl.src;
-
-	profile.releases = scrapeAll(html);
-
-	return profile;
-}
-
-async function fetchLatest(site, page = 1) {
-	const res = await http.get(`${site.url}/new-videos/${page}`);
-
-	return scrapeAll(res.body.toString(), site);
-}
-
-async function fetchScene(url, site) {
-	const useGallery = true;
-
-	// TODO: fall back on screenshots when gallery is not available
-	const res = useGallery
-		? await http.get(`${url}/gallery#gallery`)
-		: await http.get(`${url}/screenshots#screenshots`);
-
-	return scrapeScene(res.body.toString(), url, site, useGallery);
-}
-
-async function fetchProfile({ name: actorName }) {
-	const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
-	const data = res.body;
-
-	const result = data.terms.find((item) => item.type === 'model');
-
-	if (result) {
-		const bioRes = await http.get(result.url);
-		const html = bioRes.body.toString();
-
-		return scrapeProfile(html, result.url, actorName);
-	}
-
-	return null;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchProfile,
-	fetchScene,
-};

src/scrapers/assylum.js

@@ -1,6 +1,9 @@
 'use strict';
 
-const { get, geta, ctxa } = require('../utils/q');
+const unprint = require('unprint');
+
+// paths are relative, read raw attributes instead of resolved URLs
+const raw = { forceGetAttribute: true };
 
 function extractActors(actorString) {
 	return actorString
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
 }
 
 function scrapeLatest(scenes, site, models) {
-	return scenes.map(({ qu }) => {
+	return scenes.map(({ query }) => {
 		const release = {};
 
-		const pathname = qu.url('a.itemimg').slice(1);
+		const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
 		[release.entryId] = pathname.split('/').slice(-1);
 		release.url = `${site.url}${pathname}`;
 
-		release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
+		release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
-		release.description = qu.q('.mas_longdescription', true);
+		release.description = query.content('.mas_longdescription');
-		release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
+		release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
 
-		const actorString = qu.q('.mas_description', true);
+		const actorString = query.content('.mas_description');
 		const actors = matchActors(actorString, models);
 
 		if (actors.length > 0) release.actors = actors;
 		else release.actors = extractActors(actorString);
 
-		const posterPath = qu.img('.itemimg img');
+		const posterPath = query.attribute('.itemimg img', 'src', raw);
 		release.poster = `${site.url}/${posterPath}`;
 
 		return release;
 	});
 }
 
-function scrapeScene({ html, qu }, url, site, include, models) {
+function scrapeScene({ html, query }, url, site, include, models) {
 	const release = { url };
 
 	[release.entryId] = url.split('/').slice(-1);
-	release.title = qu.q('.mas_title', true);
+	release.title = query.content('.mas_title');
-	release.description = qu.q('.mas_longdescription', true);
+	release.description = query.content('.mas_longdescription');
-	release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
 
 	if (models) {
-		const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
+		const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
 		const actors = matchActors(actorString, models);
 		if (actors.length > 0) release.actors = actors;
 		else release.actors = extractActors(actorString);
 	}
 
-	release.tags = qu.all('.tags a', true);
+	release.tags = query.contents('.tags a');
 
-	release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
+	release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
 
 	const posterIndex = 'splash:';
 	const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
 	return release;
 }
 
-function extractModels({ el }, site) {
+function extractModels({ element }, site) {
-	const models = ctxa(el, '.item');
+	const models = unprint.initAll(element, '.item');
 
-	return models.map(({ qu }) => {
+	return models.map(({ query }) => {
 		const actor = { gender: 'female' };
 
-		const avatar = qu.q('.itemimg img');
+		const avatar = query.element('.itemimg img');
 		actor.avatar = `${site.url}/${avatar.src}`;
 		actor.name = avatar.alt
 			.split(':').slice(-1)[0]
 			.replace(/xtreme girl|nurse/ig, '')
 			.trim();
 
-		const actorPath = qu.url('.itemimg');
+		const actorPath = query.attribute('.itemimg', 'href', raw);
 		actor.url = `${site.url}${actorPath.slice(1)}`;
 
 		return actor;
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
 
 async function fetchModels(site, page = 1, accModels = []) {
 	const url = `${site.url}/?models/${page}`;
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const models = extractModels(res.item, site);
+		const models = extractModels(res.context, site);
-		const nextPage = res.item.qa('.pagenumbers', true)
+		const nextPage = res.context.query.contents('.pagenumbers')
 			.map((pageX) => Number(pageX))
 			.filter(Boolean) // remove << and >>
 			.includes(page + 1);
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
 
 async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
 	const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
-	const res = await geta(url, '.item');
+	const res = await unprint.get(url, { selectAll: '.item' });
 
-	return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
+	return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
 }
 
 async function fetchScene(url, site, release, include, beforeFetchLatest) {
 	const models = beforeFetchLatest || await fetchModels(site);
-	const res = await get(url);
+	const res = await unprint.get(url);
 
-	return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
+	return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
 }
 
 module.exports = {

src/scrapers/aziani.js

@@ -6,11 +6,13 @@ const { decode } = require('html-entities');
 const slugify = require('../utils/slugify');
 const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
 
+const tagKeys = ['tags', 'category', 'categories'];
+
 function scrapeScene(data, channel, parameters) {
 	const release = {};
 
 	release.entryId = data.cms_set_id;
-	release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
+	release.url = `${parameters.scene || channel.url}${parameters.scenePath || '/video'}/${parameters.entryIdSlug ? data.slug : data.cms_set_id}`;
 
 	release.title = data.name;
 	release.description = data.description;
@@ -20,7 +22,7 @@ function scrapeScene(data, channel, parameters) {
 
 	release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models' || dataType.data_type === 'Talent')?.data_values.map((actor) => ({
 		name: actor.name,
-		url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
+		url: `${channel.url}${parameters.modelPath || '/model'}/${parameters.entryIdSlug && actor.slug ? actor.slug : actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
 	}));
 
 	release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
@@ -29,7 +31,7 @@ function scrapeScene(data, channel, parameters) {
 	}));
 
 	release.tags = data.data_types
-		.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
+		.filter((dataType) => tagKeys.includes(dataType.data_type?.toLowerCase()))
 		.flatMap((tags) => tags.data_values.map((tag) => tag.name));
 
 	const poster = data.preview_formatted.thumb;
@@ -43,7 +45,7 @@ function scrapeScene(data, channel, parameters) {
 			.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
 	}
 
-	release.caps = data.content
+	release[parameters.imageType || 'caps'] = data.content
 		?.filter((item) => item.content_type === 'image' && item.content?.[0])
 		.map((item) => [
 			item.content[0],
@@ -66,13 +68,14 @@ function scrapeScene(data, channel, parameters) {
 		release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
 	}
 
-	release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
+	release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '') || null;
 
 	return release;
 }
 
 async function getBlockId(slug, dataSource, entity, parameters) {
-	const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
+	const res = await unprint.get(`${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -109,9 +112,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
 		data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
 	}).toString();
 
-	const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
+	const url = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
 
 	const res = await unprint.get(url, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: channel.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -126,20 +130,21 @@ async function fetchLatest(channel, page = 1, { parameters }) {
 }
 
 async function fetchScene(url, entity, _baseRelease, { parameters }) {
-	const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
+	const entryId = new URL(url).pathname.match(/\/videos?\/([\w-]+)/)[1];
 
 	if (!entryId) {
 		return null;
 	}
 
-	const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
+	const blockId = await getBlockId(`${parameters.videos || '/video'}/${parameters.entryIdSlug ? ':slug' : ':id'}`, entryId, entity, parameters);
 
 	if (!blockId) {
 		return null;
 	}
 
 	const query = new URLSearchParams({
-		cms_set_ids: entryId,
+		cms_set_ids: parameters.entryIdSlug ? '' : entryId,
+		...parameters.entryIdSlug ? { slug: entryId } : null,
 		cms_area_id: parameters.areaId,
 		cms_block_id: blockId,
 		content: 1,
@@ -150,9 +155,10 @@ async function fetchScene(url, entity, _baseRelease, { parameters }) {
 		content_count: 1,
 	}).toString();
 
-	const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
+	const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
 
 	const res = await unprint.get(apiUrl, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -174,14 +180,18 @@ function scrapeProfile(data, entity, parameters) {
 		decode(detail.value || detail.content_formatted || detail.content),
 	]));
 
-	profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
+	profile.url = `${entity.url}/model/${parameters.entryIdSlug ? data.slug : data.cms_data_value_id}`;
 	profile.entryId = data.cms_data_value_id;
 
 	profile.description = data.description;
 
 	profile.gender = bio.gender?.toLowerCase();
 	profile.age = bio.age;
-	profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
+	profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ })
+		|| unprint.extractDate(bio.date_of_birth, 'yyyy-MM-dd');
+
+	profile.birthPlace = bio.birthplace;
+	profile.ethnicity = bio.ethnicity;
 
 	profile.measurements = bio.measurements;
 	profile.height = feetInchesToCm(bio.height);
@@ -190,7 +200,9 @@ function scrapeProfile(data, entity, parameters) {
 	profile.hairColor = bio.hair_color;
 	profile.eyeColor = bio.eye_color;
 
-	const avatar = bio.thumbnail?.image[0];
+	const avatar = bio.thumbnail?.image?.[0] // aziani
+		|| bio.biopic?.image?.[0] // cosplayground profile
+		|| bio.headshot?.image?.[0]; // cosplayground headshot
 
 	if (avatar) {
 		profile.avatar = {
@@ -201,36 +213,41 @@ function scrapeProfile(data, entity, parameters) {
 		};
 	}
 
+	profile.socials = [bio.onlyfans, bio.instagram].filter(Boolean);
+
 	return profile;
 }
 
 async function fetchProfile({ name, url }, { entity, parameters }) {
-	if (!url) {
+	if (!url && !parameters.entryIdSlug) {
-		// no easy search option
+		// no easy search option and numeric ID is non-derivable
 		return null;
 	}
 
-	const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
+	const actorId = url && new URL(url).pathname.match(/models?\/(\d+)/)?.[1];
 
-	if (!actorId) {
+	if (!actorId && !parameters.entryIdSlug) {
 		return null;
 	}
 
-	const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
+	const slug = slugify(name);
+	const blockId = await getBlockId(`${parameters.modelPath || '/model'}/${parameters.entryIdSlug ? ':slug' : 'id'}`, { models: name, id: actorId, slug }, entity, parameters);
 
 	if (!blockId) {
 		return null;
 	}
 
 	const query = new URLSearchParams({
-		cms_data_value_ids: actorId,
+		cms_data_value_ids: actorId || '',
+		...parameters.entryIdSlug ? { slug } : null,
 		cms_block_id: blockId,
 		cms_data_type_id: 4,
 	}).toString();
 
-	const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
+	const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/data-values?${query}`;
 
 	const res = await unprint.get(apiUrl, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': entity.parameters.areaId,

src/scrapers/badoink.js

@@ -1,6 +1,7 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
+
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('a', 'href', { origin: channel.url });
+		release.url = query.url('a', { origin: channel.url });
-		release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
+		release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
 
-		release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
+		release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
-		release.description = query.cnt('.video-card-description');
+		release.description = query.content('.video-card-description');
 
-		release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
+		release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
-		release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
+		release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
 
 		release.actors = query.all('.video-card-details--cast a').map((el) => ({
-			name: qu.query.cnt(el),
+			name: unprint.query.content(el),
-			url: qu.query.url(el, null, 'href', { origin: channel.url }),
+			url: unprint.query.url(el, null, { origin: channel.url }),
 		}));
 
-		release.tags = query.cnts('.video-card-tags a');
+		release.tags = query.contents('.video-card-tags a');
 
-		const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
+		const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
 		release.poster = getPoster(posterSources);
 
 		release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
 	});
 }
 
-function scrapeScene({ query }, url, channel) {
+function scrapeScene({ query }, { url, entity: channel }) {
 	const release = {};
 
 	release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
 
-	release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
+	release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
-	release.description = query.cnt('.video-description');
+	release.description = query.content('.video-description');
 
-	release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
-	release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
+	release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
 
 	release.actors = query.all('.video-actors a').map((el) => ({
-		name: qu.query.cnt(el),
+		name: unprint.query.content(el),
-		url: qu.query.url(el, null, 'href', { origin: channel.url }),
+		url: unprint.query.url(el, null, { origin: channel.url }),
 	}));
 
-	release.tags = query.cnts('.video-tags a');
+	release.tags = query.contents('.video-tags a');
 
-	const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
+	const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
 	release.poster = getPoster(posterSources);
 
-	const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
+	const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
-	const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
+	const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
 	const lastPhoto = photos.slice(-1)[0];
 	const photosHaveIndex = /_\d+\./.test(lastPhoto);
 
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
 
 	const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
 		...acc,
-		[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
+		[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
 	}), {});
 
-	profile.description = query.cnt('.girl-details-bio');
+	profile.description = query.content('.girl-details-bio');
 
 	profile.age = bio.age;
 	profile.birthPlace = bio.country;
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
 	profile.hairColor = bio.hair;
 	profile.eyes = bio.eyes;
 
-	const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
+	const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
 
 	profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
 	profile.social = query.urls('.girl-details-social-media-list a');
 
-	profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
+	profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
 
 	return profile;
 }
 
 async function fetchLatest(channel, page) {
-	const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
+	const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
-		Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
+		selectAll: '.video-card',
+		cookies: {
+			affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
+		},
 	});
 
 	if (res.ok) {
-		return scrapeAll(res.items, channel);
+		return scrapeAll(res.context, channel);
 	}
 
 	return res.status;
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
 
 async function fetchProfile(baseActor, { entity }) {
 	const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeProfile(res.item, url, entity);
+		return scrapeProfile(res.context, url, entity);
 	}
 
 	return res.status;

src/scrapers/bang.js

@@ -62,7 +62,7 @@ function scrapeAll(scenes, entity) {
 		release.title = query.content('a > span.block');
 		release.date = query.date('a + div', 'MMM DD, YYYY');
 
-		release.duration = query.duration('[data-videopreview-target="duration"]', { attribute: 'data-content' });
+		release.duration = query.duration('[data-videopreview-target="duration"], .video_duration', { attribute: 'data-content' });
 
 		release.actors = query.elements('a + div a[href*="/pornstar"]').map((el) => ({
 			name: unprint.query.content(el),
@@ -70,30 +70,29 @@ function scrapeAll(scenes, entity) {
 		}));
 
 		const poster = query.img('img[data-videopreview-target="image"]');
-		const posterUrl = new URL(poster);
 
 		if (poster) {
-			release.poster = [
+			try {
-				`${posterUrl.origin}${posterUrl.pathname}`,
+				const posterUrl = new URL(poster);
-				posterUrl.href,
+
-			].map((src) => ({
+				release.poster = [
-				src,
+					`${posterUrl.origin}${posterUrl.pathname}`,
-				expectType: {
+					posterUrl.href,
-					'application/octet-stream': 'image/jpeg',
+				].map((src) => ({
-				},
+					src,
-			}));
+					expectType: {
+						'application/octet-stream': 'image/jpeg',
+					},
+				}));
+			} catch (error) {
+				// nothing needed
+			}
 		}
 
-		const videoData = query.json('a', { attribute: 'data-videopreview-sources-value' });
+		release.teaser = [
-
+			query.url('a[data-videopreview-mp4-large-value]', { attribute: 'data-videopreview-mp4-large-value' }),
-		if (videoData) {
+			query.url('a[data-videopreview-mp4-value]', { attribute: 'data-videopreview-mp4-value' }),
-			release.teaser = [
+		].filter(Boolean);
-				videoData.mp4_large,
-				videoData.webm_large,
-				videoData.mp4,
-				videoData.webm,
-			];
-		}
 
 		return release;
 	});
@@ -150,7 +149,7 @@ async function scrapeScene({ query }, { url, entity }) {
 		}));
 	}
 
-	release.tags = query.contents('div[data-controller] .genres');
+	release.tags = query.contents('a.genres');
 
 	const sourcesData = query.json('.video-container [data-videopreview-sources-value]', { attribute: 'data-videopreview-sources-value' });
 
@@ -175,7 +174,7 @@ async function scrapeScene({ query }, { url, entity }) {
 		|| query.attribute('meta[property="og:video"]')
 		|| query.video('video[data-videocontainer-target] source');
 
-	release.photos = query.sourceSets('div[data-controller] a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
+	release.photos = query.sourceSets('a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
 		src,
 		expectType: {
 			'application/octet-stream': 'image/jpeg',

src/scrapers/czechav.js

@@ -1,29 +1,29 @@
 'use strict';
 
-const qu = require('../utils/q');
+const unprint = require('unprint');
 
 function scrapeAll(scenes, channel) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('a', 'href', { origin: channel.url });
+		release.url = query.url('a', { origin: channel.url });
 		release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
 
-		release.title = query.cnt('.nice-title');
+		release.title = query.content('.nice-title');
 
 		release.poster = query.img('img.poster');
-		release.photos = query.q('img.poster', 'data-urls')?.split(' ');
+		release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
-		release.teaser = query.video('img.poster', 'data-thumbnail');
+		release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
 
 		if (query.exists('.info')) {
 			release.date = query.date('.updated', 'MM/DD/YYYY');
 
 			release.actors = [{
-				name: query.cnt('.name'),
+				name: query.content('.name'),
-				age: query.cnt('.age'),
+				age: query.content('.age'),
 			}];
 
-			release.tags = query.cnts('.tags a');
+			release.tags = query.contents('.tags a');
 		}
 
 		return release;
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
 
 	release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
 
-	release.title = query.cnt('.nice-title');
+	release.title = query.content('.nice-title');
-	release.description = query.cnt('.desc-text p');
+	release.description = query.content('.desc-text p');
 
-	release.tags = query.cnts('.tags a');
+	release.tags = query.contents('.tags a');
 
 	release.poster = query.poster();
 	release.photos = query.imgs('.thumbnails img, .thumb img');
 
 	// some videos may not have unique trailers, but many do
-	release.trailer = query.video();
+	release.trailer = query.video('source');
 
-	release.formats = query.cnts('.download-quality option');
+	release.formats = query.contents('.download-quality option');
 
 	return release;
 }
 
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/tour/videos/page-${page}`;
-	const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
+	const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
 
 	if (res.ok) {
-		return scrapeAll(res.items, channel);
+		return scrapeAll(res.context, channel);
 	}
 
 	return res.status;
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, channel);
+		return scrapeScene(res.context, url, channel);
 	}
 
 	return res.status;

src/scrapers/dorcel.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const unprint = require('unprint');
-const cookie = require('cookie');
 
 const slugify = require('../utils/slugify');
 
@@ -54,16 +53,14 @@ function scrapeAll(scenes, channel) {
 
 async function beforeFetchLatest(channel) {
 	// scene page only seems to accept language preferences from session
-	const { res } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
+	const { cookies } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
 		headers: {
 			'X-Requested-With': 'XMLHttpRequest',
 			'Accept-Language': 'en-US,en', // fetch English rather than French titles
 		},
 	});
 
-	const sessionCookie = cookie.parse(res.headers['set-cookie'][0])?.dorcelclub;
+	return `dorcelclub=${cookies.dorcelclub}`;
-
-	return `dorcelclub=${sessionCookie}`;
 }
 
 async function fetchLatest(channel, page = 1, _options, { beforeFetchLatest: sessionCookie }) {

src/scrapers/elevatedx.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const format = require('template-format');
+const unprint = require('unprint');
 
-const qu = require('../utils/q');
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
 	if (release.date && release.url) {
 		const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
 
-		return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
+		return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
 	}
 
 	if (release.date && release.title) {
-		return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
+		return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
 	}
 
 	return null;
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
 
 function extractPoster(posterPath, channel, baseRelease) {
 	if (posterPath && !/400.jpg/.test(posterPath)) {
-		const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
+		const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
 		const posterSources = [
 			poster,
 			// upscaled
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
 	return [baseRelease?.poster || null, []];
 }
 
-function getImageWithFallbacks(q, selector, site, el) {
+function getImageWithFallbacks(query, selector, site, el) {
 	const sources = el
 		? [
-			q(el, selector, 'src0_3x'),
+			unprint.query.attribute(el, selector, 'src0_3x'),
-			q(el, selector, 'src0_2x'),
+			unprint.query.attribute(el, selector, 'src0_2x'),
-			q(el, selector, 'src0_1x'),
+			unprint.query.attribute(el, selector, 'src0_1x'),
 		]
 		: [
-			q(selector, 'src0_3x'),
+			query.attribute(selector, 'src0_3x'),
-			q(selector, 'src0_2x'),
+			query.attribute(selector, 'src0_2x'),
-			q(selector, 'src0_1x'),
+			query.attribute(selector, 'src0_1x'),
 		];
 
-	return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
+	return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
 }
 
 function scrapeAllClassic(scenes, channel) {
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
 		const release = {};
 
 		release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
-		release.entryId = query.url('.updateThumb img', 'alt');
+		release.entryId = query.attribute('.updateThumb img', 'alt');
 
-		release.title = query.cnt('.updateInfo h5 a');
+		release.title = query.content('.updateInfo h5 a');
 
-		release.actors = query.cnts('.tour_update_models a');
+		release.actors = query.contents('.tour_update_models a');
 		release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
 
-		release.poster = query.img('.updateThumb img');
+		release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
 
-		const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
+		const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
 
 		if (trailer) {
 			release.trailer = `${channel.url}${trailer}`;
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
+		release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
 		release.url = query.url('h4 a');
 
 		release.date = query.date('.more-info-div', 'MMM D, YYYY');
-		release.duration = query.dur('.more-info-div');
+		release.duration = query.duration('.more-info-div');
 
-		const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
+		// raw path, prefixed with media URL below
+		const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
 
 		if (posterPath) {
 			const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 			];
 		}
 
-		release.teaser = query.video();
+		release.teaser = query.video('source');
 
 		// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
 		release.entryId = deriveEntryId(release);
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
 	});
 }
 
-function scrapeSceneClassic({ query, html }, url, channel) {
+function scrapeSceneClassic({ query, html }, { entity: channel }) {
 	const release = {};
 
-	release.title = query.q('.updatesBlock h2', true);
+	release.title = query.content('.updatesBlock h2');
-	release.poster = query.meta('property="og:image"');
+	release.poster = query.attribute('meta[property="og:image"]', 'content');
 	release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
 
 	const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
 	return release;
 }
 
-function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
+function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
 	const release = {};
 
-	release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
+	release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
 	release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
 
-	release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
-	release.duration = query.dur('.update-info-row:nth-child(2)');
+	release.duration = query.duration('.update-info-row:nth-child(2)');
-	release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
+	release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
 
 	release.actors = query.all('.models-list-thumbs a').map((el) => ({
-		name: query.cnt(el, 'span'),
+		name: unprint.query.content(el, 'span'),
-		avatar: getImageWithFallbacks(query.q, 'img', entity, el),
+		avatar: getImageWithFallbacks(query, 'img', entity, el),
-		url: query.url(el, null),
+		url: unprint.query.url(el, null),
 	}));
 
-	release.tags = query.all('.tags a', true);
+	release.tags = query.contents('.tags a');
 
-	const posterPath = query.q('.player-thumb img', 'src0_1x');
+	const posterPath = query.attribute('.player-thumb img', 'src0_1x');
 	const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
 
 	[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
 
 	if (trailer) {
-		release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
+		release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
 	}
 
-	const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
+	const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
 	if (stars) release.stars = Number(stars);
 
 	if (entity.type === 'network') {
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
 async function scrapeProfile({ query }, entity, parameters) {
 	const profile = {};
 
-	const bio = query.cnt('.model_bio, .detail-div');
+	const bio = query.content('.model_bio, .detail-div');
-	const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
+	const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
 
 	profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
-	profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
+	profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
 
 	profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
 	profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
 			avatarEl.getAttribute('src'),
 		]
 			.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
-			.map((avatar) => qu.prefixUrl(avatar, entity.url));
+			.map((avatar) => unprint.prefixUrl(avatar, entity.url));
 
 		if (avatarSources.length) profile.avatar = avatarSources;
 	}
 
 	if (parameters?.layout === 'classic') {
-		profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
+		profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
 	}
 
 	if (parameters?.layout === 'tubular') {
-		profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
+		profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
 	}
 
 	return profile;
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
 	const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
         || `${site.url}/categories/movies_${page}_d.html`;
 
-	const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
+	const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
 
 	if (!res.ok) {
 		return res.status;
 	}
 
-	return allScraper(res.items, site, preData?.uniqueReleases);
+	return allScraper(res.context, site, preData?.uniqueReleases);
 }
 
 async function fetchUpcomingClassic(channel) {
-	const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
+	const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
 
 	if (res.ok) {
-		return scrapeAllClassic(res.items, channel);
+		return scrapeAllClassic(res.context, channel);
 	}
 
 	return res.status;
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
 			return null;
 		}
 
-		const res = await qu.get(profileUrl);
+		const res = await unprint.get(profileUrl);
 
-		if (res.statusCode === 200) {
+		if (res.status === 200) {
-			return scrapeProfile(res.item, entity, parameters);
+			return scrapeProfile(res.context, entity, parameters);
 		}
 
 		return null;
@@ -290,14 +291,12 @@ module.exports = {
 		fetchProfile,
 		scrapeAll: scrapeAllClassic,
 		scrapeScene: scrapeSceneClassic,
-		deprecated: true,
 	},
 	tubular: {
 		fetchLatest: fetchLatestTubular,
 		fetchProfile,
 		scrapeAll: scrapeAllTubular,
 		scrapeScene: scrapeSceneTubular,
-		deprecated: true,
 	},
 	getImageWithFallbacks,
 };

src/scrapers/hookuphotshot.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const unprint = require('unprint');
+const { nanoid } = require('nanoid');
 
 function scrapeAll(scenes, _channel) {
 	return scenes.map(({ query }) => {
@@ -40,6 +41,12 @@ async function fetchLatest(channel, page = 1) {
 	return res.status;
 }
 
+async function beforeFetchScenes(channel) {
+	const res = await unprint.get(channel.url);
+
+	return { nats: res.cookies.nats || nanoid() };
+}
+
 function scrapeScene({ query, html }, { url, entity, baseRelease }) {
 	const release = {};
 
@@ -81,6 +88,20 @@ function scrapeScene({ query, html }, { url, entity, baseRelease }) {
 	return release;
 }
 
+async function fetchScene(url, entity, baseRelease, context) {
+	const res = await unprint.get(url, {
+		cookies: {
+			nats: context.beforeFetchScenes?.nats,
+		},
+	});
+
+	if (res.ok) {
+		return scrapeScene(res.context, { url, entity, baseRelease });
+	}
+
+	return res.status;
+}
+
 function scrapeProfile({ query }) {
 	const profile = {};
 
@@ -110,5 +131,6 @@ async function fetchProfile(actor, entity) {
 module.exports = {
 	fetchLatest,
 	fetchProfile,
-	scrapeScene,
+	fetchScene,
+	beforeFetchScenes,
 };

src/scrapers/insex.js

@@ -1,92 +1,75 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
-const http = require('../utils/http');
+
 const slugify = require('../utils/slugify');
 
 function scrapeLatest(scenes, site) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+		release.url = query.url('figure a', { origin: site.parameters.latest });
 
-		release.title = query.cnt('.has-text-weight-bold, .is-size-6');
+		release.title = query.content('.has-text-weight-bold, .is-size-6');
 		release.date = query.date('span.tag', 'YYYY-MM-DD');
-		release.actors = query.cnts('a.tag');
+		release.actors = query.contents('a.tag');
 
 		const cover = query.img('.image img');
 
 		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
 		release.covers = [cover];
 
-		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+		release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+
+		console.log(release);
 
 		return release;
 	});
 }
 
-function scrapeLatestLegacy(scenes, site) {
+async function fetchLatest(site, page = 1) {
-	return scenes.map(({ query }) => {
+	const domain = new URL(site.url).searchParams.get('d');
-		// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
-		const release = {};
 
-		const titleEl = query.q('.galleryTitleText, .articleTitleText');
+	if (!domain) {
-		const [title, ...actors] = titleEl.textContent.split('|');
+		return null;
-		const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
+	}
 
-		const url = query.url(titleEl, 'a');
+	const url = `${site.origin}/is/home.php?o=newest&d=${domain}&p=${page}`;
-		[release.entryId] = url.split('/').slice(-2);
-		release.url = `${site.url}${url}`;
 
-		if (date) {
+	const res = await unprint.get(url, {
-			release.title = title.trim();
+		selectAll: '.column.is-3-desktop',
-			release.date = date;
+		cookies: { consent: 'yes' },
-		} else {
-			//  title should contain date instead, not applicable in brief mode
-			release.title = title.slice(title.indexOf(':') + 1).trim();
-			release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
-		}
-
-		release.actors = actors.map((actor) => actor.trim());
-
-		const description = query.q('.articleCopyText', true);
-		if (description) release.description = description.slice(0, description.lastIndexOf('('));
-
-		const duration = query.dur('.articleCopyText a:nth-child(2)');
-		if (duration) release.duration = duration;
-
-		release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
-
-		const cover = query.img('a img');
-
-		release.covers = [[
-			cover.replace('_thumbnail', ''),
-			cover,
-		]];
-
-		return release;
 	});
+
+	// (UNSEXUALLY) BROKEN, abandoned sites, low priority
+
+	if (res.ok) {
+		return scrapeLatest(res.context, site);
+	}
+
+	return res.status;
 }
 
-async function scrapeScene({ query }, url, channel, parameters, session) {
+async function scrapeScene({ query }, channel, include, cookies) {
 	const release = {};
 
-	release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
+	release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
-	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
+	release.description = query.content('.has-background-black-ter > div:nth-child(4)');
 	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
 
-	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
+	release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
-	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
+	release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
 
-	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
+	release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
-	release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
+	release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
 
-	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
+	release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
 
-	release.trailer = query.video();
+	release.trailer = query.video('source');
 
-	if (!release.trailer && parameters.includeTrailers) {
+	if (!release.trailer && include.includeTrailers) {
-		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
+		// play API relies on the session cookie set by the scene page
+		const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
 
 		if (trailerRes.ok) {
 			release.trailer = trailerRes.body;
@@ -96,63 +79,13 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
 	return release;
 }
 
-function scrapeSceneLegacy({ query }, site) {
+async function fetchScene(url, site, baseRelease, include) {
-	const release = {};
+	const res = await unprint.get(url, {
-
+		cookies: { consent: 'yes' },
-	const titleEl = query.q('.articleTitleText');
+	});
-	const [title, ...actors] = titleEl.textContent.split('|');
-
-	const url = query.url(titleEl, 'a');
-	[release.entryId] = url.split('/').slice(-2);
-	release.url = `${site.url}${url}`;
-
-	release.title = title.trim();
-	release.description = query.q('.articleCopyText', true);
-
-	release.actors = actors.map((actor) => actor.trim());
-	release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
-	release.duration = query.dur('.articlePostDateText a:nth-child(2)');
-
-	const [cover, ...photos] = query.imgs('img[src*="images"]');
-	release.covers = [cover];
-	release.photos = photos;
-
-	release.poster = query.poster();
-
-	const trailer = query.trailer();
-	if (trailer) release.trailer = { src: trailer };
-
-	return release;
-}
-
-async function fetchLatest(site, page = 1) {
-	const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
-	const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
 
 	if (res.ok) {
-		return scrapeLatest(res.items, site);
+		return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
-	}
-
-	return res.status;
-}
-
-async function fetchLatestLegacy(site, page = 1) {
-	const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-	const res = await qu.get(url); // JSON containing html as a property
-
-	if (res.ok) {
-		return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
-	}
-
-	return res.status;
-}
-
-async function fetchScene(url, site, baseRelease, parameters) {
-	const session = http.session();
-	const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
-
-	if (res.ok) {
-		return scrapeScene(res.item, url, site, parameters, session);
 	}
 
 	return res.status;
@@ -161,8 +94,4 @@ async function fetchScene(url, site, baseRelease, parameters) {
 module.exports = {
 	fetchLatest,
 	fetchScene,
-	legacy: {
-		fetchLatest: fetchLatestLegacy,
-		scrapeScene: scrapeSceneLegacy,
-	},
 };

src/scrapers/jesseloadsmonsterfacials.js

@@ -1,11 +1,14 @@
 'use strict';
 
-const { get, initAll } = require('../utils/qu');
+const unprint = require('unprint');
+
+// paths are relative to /visitors/, read raw attributes instead of resolved URLs
+const raw = { forceGetAttribute: true };
 
 function scrapeLatest(scenes, dates, site) {
-	return scenes.map(({ qu }, index) => {
+	return scenes.map(({ query }, index) => {
 		const release = {};
-		const path = qu.url('a[href*="videos/"]');
+		const path = query.attribute('a[href*="videos/"]', 'href', raw);
 
 		if (path) {
 			if (/\.wmv$/.test(path)) {
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
 		}
 
 		if (dates && dates[index]) {
-			release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
+			release.date = dates[index].query.date(null, 'MM/DD/YYYY');
 		}
 
 		// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
 		release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
-			|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
+			|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
 
-		release.description = qu.q('tbody tr:nth-child(3) font', true);
+		release.description = query.content('tbody tr:nth-child(3) font');
 
-		const infoLine = qu.q('font[color="#663366"]', true);
+		const infoLine = query.content('font[color="#663366"]');
 
 		if (infoLine) {
 			release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
 		}
 
-		const poster = qu.img('img[src*="photos/"][width="400"]');
+		const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
 		release.poster = `${site.url}/visitors/${poster}`;
-		release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
+		release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
 
 		return release;
 	});
 }
 
-function scrapeScene({ qu }, url, site) {
+function scrapeScene({ query }, url, site) {
 	const release = { url };
 
 	const { pathname } = new URL(url);
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
 		return release;
 	}
 
-	const actor = qu.q('font[color="#990033"] strong', true);
+	const actor = query.content('font[color="#990033"] strong');
 	release.actors = [actor];
 
-	const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
+	const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
-	const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
+	const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
 
 	release.trailer = [
-		{
+		hdTrailer && {
 			src: `${site.url}/visitors/videos/${hdTrailer}`,
 			quality: 1080,
 		},
-		{
+		sdTrailer && {
 			src: `${site.url}/visitors/videos/${sdTrailer}`,
 			quality: 270,
 		},
-	];
+	].filter(Boolean);
 
 	return release;
 }
 
 async function fetchLatest(site, page = 1) {
 	const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (!res.ok) {
 		return res.status;
 	}
 
-	const { el } = res.item;
+	const { element } = res.context;
 
-	const scenes = initAll(el, 'table[width="880"]');
+	const scenes = unprint.initAll(element, 'table[width="880"]');
-	const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
+	const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
 
 	return scrapeLatest(scenes, dates, site);
 }
 
 async function fetchScene(url, site) {
-	const res = await get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, url, site);
+		return scrapeScene(res.context, url, site);
 	}
 
 	return res.status;

src/scrapers/manyvids.js

@@ -26,6 +26,7 @@ async function getTrailer(videoId) {
 	return null;
 }
 
+/*
 function getModelUrl(modelData) {
 	const slug = modelData.slug || modelData.profileUrl?.match(/\/profile\/\d+\/(\w+)/i)?.[1] || null;
 
@@ -35,6 +36,7 @@ function getModelUrl(modelData) {
 
 	return null;
 }
+*/
 
 async function scrapeScene(data, context) {
 	const release = {};
@@ -61,6 +63,7 @@ async function scrapeScene(data, context) {
 	release.date = new Date(data.launchDate);
 	release.duration = unprint.extractDuration(data.duration || data.videoDuration);
 
+	/* many channels operate more like a studio than a performer, so the channel information doesn't represent a person
 	const model = data.creator || data.model;
 
 	release.actors = [{
@@ -69,6 +72,7 @@ async function scrapeScene(data, context) {
 		avatar: model.avatar?.url || model.avatar,
 		url: getModelUrl(model),
 	}];
+	*/
 
 	release.tags = [
 		data.is3D && '3d',
@@ -86,18 +90,44 @@ async function scrapeScene(data, context) {
 
 	release.qualities = [data.height].filter(Boolean);
 
-	console.log(release);
-
 	return release;
 }
 
-async function fetchLatest(channel, page = 1) {
+const profileIdRegex = /\/profile\/(\d+)/i;
-	const idRes = await unprint.request(channel.url, {
-		method: 'head',
-		followRedirects: 0,
-	});
 
-	const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1];
+async function getProfileId(channel) {
+	if (channel.url) {
+		const { hostname, pathname } = new URL(channel.url);
+
+		if (profileIdRegex.test(pathname)) { // full profile URL
+			const urlProfileId = new URL(channel.url).pathname.match(profileIdRegex)?.[1];
+
+			if (urlProfileId) {
+				return urlProfileId;
+			}
+		}
+
+		if (/[\w-]+.manyvids.com/.test(hostname)) { // short profile URL
+			const idRes = await unprint.request(channel.url, {
+				method: 'head',
+				followRedirects: 0,
+			});
+
+			const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
+
+			if (profileId) {
+				return profileId;
+			}
+		}
+
+		// sadly we can't reliably reconstruct short profile URL from the slug
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const profileId = await getProfileId(channel);
 
 	if (profileId) {
 		const res = await unprint.get(`https://api.manyvids.com/store/videos/${profileId}?sort=newest&limit=100&page=${page}`);
@@ -154,8 +184,6 @@ function scrapeProfile(data) {
 		profile.dateOfBirth = dateOfBirth;
 	}
 
-	console.log(profile);
-
 	return profile;
 }
 
@@ -166,7 +194,7 @@ async function getActorUrl(actor) {
 			followRedirects: 0,
 		});
 
-		const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1];
+		const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
 
 		if (profileId) {
 			return `https://api.manyvids.com/profile/profile/${profileId}`;

src/scrapers/nubiles.js

@@ -22,12 +22,13 @@ function stripQuery(link) {
 
 async function getPhotos(albumUrl) {
 	// aggressive Turnstile protection calls for browser
-	const res = await unprint.browser(albumUrl, {
+	const res = await unprint.get(albumUrl, {
 		selectAll: '.photo-thumb',
+		interface: 'request',
 	});
 
 	if (res.ok) {
-		return res.context.map(({ query }) => query.sourceSet('source, picture img')).filter(Boolean);
+		return res.context.map(({ query }) => query.img('picture img')).filter(Boolean);
 	}
 
 	return [];
@@ -53,14 +54,10 @@ function scrapeAll(scenes, entity) {
 			release.actors = query.content('.models', { trim: false })?.trim().split(/\s{2,}/);
 		}
 
-		const poster = query.sourceSet('img', 'data-srcset')?.[0]
+		// old 640px to 1280px replace won't work anymore with token in URL
+		release.poster = query.sourceSet('img', 'data-srcset')?.[0]
 			|| query.sourceSet('img', 'data-src')?.[0];
 
-		release.poster = poster && [
-			poster.replace('_640', '_1280'),
-			poster,
-		];
-
 		release.stars = query.number('.rating');
 		release.likes = query.number('.likes');
 
@@ -73,7 +70,8 @@ function scrapeAll(scenes, entity) {
 async function fetchLatest(site, page = 1) {
 	const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
 
-	const res = await unprint.browser(url, {
+	const res = await unprint.get(url, {
+		interface: 'request',
 		selectAll: '.content-grid-item',
 	});
 
@@ -88,7 +86,8 @@ async function fetchUpcoming(site) {
 	if (site.parameters?.upcoming) {
 		const url = `${site.url}/video/upcoming`;
 
-		const res = await unprint.browser(url, {
+		const res = await unprint.get(url, {
+			interface: 'request',
 			selectAll: '.content-grid-item',
 		});
 
@@ -140,7 +139,9 @@ async function scrapeScene({ query }, { url, entity, include }) {
 }
 
 async function fetchScene(url, entity, _baseRelease, include) {
-	const res = await unprint.browser(url);
+	const res = await unprint.get(url, {
+		interface: 'request',
+	});
 
 	if (res.ok) {
 		return scrapeScene(res.context, { url, entity, include });
@@ -184,7 +185,9 @@ async function findModel(actor, entity) {
 
 	const url = `${origin}/model/alpha/${firstLetter}`;
 
-	const resModels = await unprint.browser(url);
+	const resModels = await unprint.get(url, {
+		interface: 'request',
+	});
 
 	if (!resModels.ok) {
 		return resModels.status;
@@ -214,7 +217,9 @@ async function fetchProfile(actor, { entity }) {
 	const model = await findModel(actor, entity);
 
 	if (model) {
-		const resModel = await unprint.browser(model.url);
+		const resModel = await unprint.get(model.url, {
+			interface: 'request',
+		});
 
 		if (resModel.ok) {
 			return scrapeProfile(resModel.context, model.avatar);

src/scrapers/perfectgonzo.js

@@ -1,15 +1,11 @@
 'use strict';
 
-const blake2 = require('blake2');
+const { blake2b } = require('@noble/hashes/blake2.js');
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
 
 function getHash(identifier) {
-	const hash = blake2.createHash('blake2b', { digestLength: 8 });
+	return Buffer.from(blake2b(Buffer.from(identifier), { dkLen: 8 })).toString('hex');
-
-	hash.update(Buffer.from(identifier));
-
-	return hash.digest('hex');
 }
 
 function extractMaleModelsFromTags(tagContainer) {
@@ -45,8 +41,8 @@ async function scrapeLatest(scenes, site) {
 	return scenes.map(({ query }) => {
 		const release = {};
 
-		release.title = query.q('a', 'title');
+		release.title = query.attribute('a', 'title');
-		release.url = query.url('a', 'href', { origin: site.url });
+		release.url = query.url('a', { origin: site.url });
 		release.date = query.date('.nm-date', 'MM/DD/YYYY');
 
 		const slug = new URL(release.url).pathname.split('/')[2];
@@ -56,7 +52,7 @@ async function scrapeLatest(scenes, site) {
 
 		[release.poster, ...release.photos] = query.imgs('.bloc-link img');
 
-		release.tags = query.cnts('.dropdown ul a').slice(1);
+		release.tags = query.contents('.dropdown ul a').slice(1);
 		release.duration = query.duration('.dropdown p:first-child');
 
 		return release;
@@ -66,22 +62,22 @@ async function scrapeLatest(scenes, site) {
 async function scrapeScene({ query }, site, url) {
 	const release = { url, site };
 
-	release.title = query.cnt('#movie-header h2');
+	release.title = query.content('#movie-header h2');
-	release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+	release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
 
-	release.description = query.cnt('.container .mg-md');
+	release.description = query.content('.container .mg-md');
 	release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
 
-	release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
+	release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
-	release.tags = query.cnts('.tag-container a');
+	release.tags = query.contents('.tag-container a');
 
-	const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
+	const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
 	if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
 
 	release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
 	release.poster = query.poster();
 
-	const trailer = query.trailer();
+	const trailer = query.video('source');
 	if (trailer) release.trailer = { src: trailer };
 
 	if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
@@ -99,20 +95,20 @@ async function scrapeScene({ query }, site, url) {
 
 async function fetchLatest(site, page = 1) {
 	const url = `${site.url}/movies/page-${page}`;
-	const res = await qu.getAll(url, '#content-main [class^="item"]');
+	const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
 
 	if (res.ok) {
-		return scrapeLatest(res.items, site);
+		return scrapeLatest(res.context, site);
 	}
 
 	return res.status;
 }
 
 async function fetchScene(url, channel) {
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		return scrapeScene(res.item, channel, url);
+		return scrapeScene(res.context, channel, url);
 	}
 
 	return res.status;

src/scrapers/pinkyxxx.js

@@ -1,24 +1,42 @@
 'use strict';
 
-const qu = require('../utils/qu');
+const unprint = require('unprint');
-const http = require('../utils/http');
+
+function parseJson(res) {
+	// admin-ajax responds with JSON labeled as text/html
+	if (typeof res.data === 'object') {
+		return res.data;
+	}
+
+	try {
+		return JSON.parse(res.body);
+	} catch (_error) {
+		return null;
+	}
+}
 
 async function getTrailer(entryId, channel) {
-	const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
+	const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
 		action: 'vidplay',
 		vid: entryId,
 		type: 'promo',
 	}, {
-		encodeJSON: false,
+		form: true,
-		decodeJSON: true,
 	});
 
-	if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
+	const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
+
+	if (!videoLinks?.size1) {
 		return null;
 	}
 
-	const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
+	const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
-	const videoScriptRes = await http.get(scriptUrl);
+
+	if (!scriptUrl) {
+		return null;
+	}
+
+	const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
 
 	if (!videoScriptRes.ok) {
 		return null;
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
 	const videoData = videoDataString && JSON.parse(videoDataString);
 
 	const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
-	const posterUrlEl = videoScriptRes.document.createElement('span');
+	const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
-
-	posterUrlEl.innerHTML = posterUrlString;
 
 	return {
 		trailer: videoData?.url || null,
-		poster: posterUrlEl.textContent || null,
+		poster: poster || null,
 	};
 }
 
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
 		release.title = scene.title;
 		release.description = scene.description;
 
-		release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
+		release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
 
 		release.poster = scene.poster[0];
 		release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
 	});
 }
 
-async function scrapeScene({ query }, channel, baseScene) {
+async function scrapeScene({ query }, { entity, baseRelease }) {
 	const release = {};
 
-	release.title = query.cnt('.title');
+	release.title = query.content('.title');
-	release.description = query.cnt('.description');
+	release.description = query.content('.description');
 
-	release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
+	release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
 
-	const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
+	const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
 
-	release.poster = baseScene?.poster || poster;
+	release.poster = baseRelease?.poster || poster;
-	release.photos = poster && baseScene?.poster
+	release.photos = poster && baseRelease?.poster
-		? [poster, ...baseScene.photos]
+		? [poster, ...baseRelease.photos]
-		: baseScene?.photos;
+		: baseRelease?.photos;
 
 	release.trailer = trailer;
 
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
 }
 
 async function fetchLatest(channel, page) {
-	const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
+	const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
 		action: 'vls',
 		limit: 10,
 		offset: (page - 1) * 10,
 	}, {
-		encodeJSON: false,
+		form: true,
-		decodeJSON: true,
 	});
 
-	if (res.ok) {
+	const data = res.ok && parseJson(res);
-		return scrapeAll(res.body.listings);
+
+	if (data?.listings) {
+		return scrapeAll(data.listings);
 	}
 
-	return res.status;
+	return res.ok ? null : res.status;
-}
-
-async function fetchScene(url, channel, baseScene) {
-	const res = await qu.get(url);
-
-	if (res.ok) {
-		return scrapeScene(res.item, channel, baseScene);
-	}
-
-	return res.status;
 }
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
+	scrapeScene,
 };

src/scrapers/radical.js

@@ -1,9 +1,8 @@
 'use strict';
 
 const unprint = require('unprint');
-const mime = require('mime');
+const mime = require('mime').default;
 
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -132,10 +131,10 @@ async function fetchLatest(channel, page, { parameters }) {
 		? `${channel.parent.url}/_next/data/${endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
 		: `${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
 
-	const res = await http.get(url);
+	const res = await unprint.get(url);
 
-	if (res.ok && res.body.pageProps?.contents?.data) {
+	if (res.ok && res.data.pageProps?.contents?.data) {
-		return scrapeAll(res.body.pageProps.contents.data, channel, parameters);
+		return scrapeAll(res.data.pageProps.contents.data, channel, parameters);
 	}
 
 	return res.status;
@@ -170,12 +169,12 @@ async function fetchScene(url, channel, baseScene, { parameters }) {
 
 	const slug = new URL(url).pathname.split('/').at(-1);
 	const endpoint = await fetchEndpoint(channel);
-	const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
+	const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
 
-	if (res.ok && res.body.pageProps?.content) {
+	if (res.ok && res.data.pageProps?.content) {
 		return scrapeScene({
-			...res.body.pageProps.content,
+			...res.data.pageProps.content,
-			serie: res.body.pageProps.playlist?.data[0],
+			serie: res.data.pageProps.playlist?.data[0],
 		}, channel, parameters);
 	}
 
@@ -215,10 +214,10 @@ function scrapeProfile(data, channel, scenes, parameters) {
 
 async function fetchProfile(actor, { channel, parameters }) {
 	const endpoint = await fetchEndpoint(channel);
-	const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
+	const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
 
-	if (res.ok && res.body.pageProps?.model) {
+	if (res.ok && res.data.pageProps?.model) {
-		return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
+		return scrapeProfile(res.data.pageProps.model, channel, res.data.pageProps.model_contents, parameters);
 	}
 
 	return res.status;

src/scrapers/releases.js

@@ -39,7 +39,6 @@ const karups = require('./karups');
 const kellymadison = require('./kellymadison');
 const kink = require('./kink');
 const mariskax = require('./mariskax');
-// const analvids = require('./analvids');
 const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
@@ -62,6 +61,7 @@ const pornworld = require('./pornworld');
 const privateNetwork = require('./private'); // reserved keyword
 const radical = require('./radical');
 const rickysroom = require('./rickysroom');
+const rsghmedia = require('./rsghmedia');
 const sexlikereal = require('./sexlikereal');
 const score = require('./score');
 const sicflics = require('./sicflics');
@@ -73,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
 const testedefudelidade = require('./testedefudelidade');
 const theflourish = require('./theflourish');
 const tokyohot = require('./tokyohot');
-// const topwebmodels = require('./topwebmodels');
 const traxxx = require('./traxxx');
 const virtualtaboo = require('./virtualtaboo');
 const vixen = require('./vixen');
@@ -102,6 +101,7 @@ module.exports = {
 	acam,
 	analvids: pornbox,
 	pornbox,
+	cosplayground: aziani,
 	kellymadison,
 	kink,
 	'8kmembers': kellymadison,
@@ -143,6 +143,7 @@ module.exports = {
 	interracialpass: hush,
 	inthecrack,
 	jerkaoke: modelmedia,
+	jenniferwhite: adultempire,
 	jesseloadsmonsterfacials,
 	julesjordan,
 	karups,
@@ -170,6 +171,7 @@ module.exports = {
 	private: privateNetwork,
 	radical,
 	rickysroom,
+	rsghmedia,
 	score,
 	sexlikereal,
 	sicflics,

src/scrapers/rickysroom.js

@@ -1,31 +1,36 @@
 'use strict';
 
-const qu = require('../utils/q');
+const unprint = require('unprint');
+
 const { lbsToKg, feetInchesToCm } = require('../utils/convert');
 
 function scrapeScene(data, channel) {
 	const release = {};
 
 	release.entryId = data.id;
-	release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
+	release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
 
 	release.title = data.title;
 	release.description = data.description;
-	release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
+	release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
-	release.duration = qu.durationToSeconds(data.videos_duration);
+	release.duration = unprint.extractDuration(data.videos_duration);
 
 	release.actors = data?.models_thumbs.map((model) => ({
 		name: model.name,
-		url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
+		url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
 		avatar: model.thumb,
 	})) || data.models;
 
 	release.tags = data.tags;
 
 	release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
-	release.photos = data.previews?.full
+
-		.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+	// inconsistent data structure. if there is an album, it's previews: { full: [...] }, if it's empty, it's previews: []
-		.concat(data.thumbs); // screenshots
+	if (Array.isArray(data.previews?.full)) {
+		release.photos = data.previews?.full
+			.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+			.concat(data.thumbs); // screenshots
+	}
 
 	release.trailer = data.trailer_url;
 	release.teaser = data.special_thumbnails;
@@ -36,39 +41,14 @@ function scrapeScene(data, channel) {
 	return release;
 }
 
-function scrapeProfile(data, scenes, entity) {
-	const profile = {};
-
-	profile.entryId = data.id;
-	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
-
-	profile.description = data.Bio || data.bio;
-	profile.birthPlace = data.Born || data.born;
-	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
-
-	profile.measurements = data.Measurements || data.Measurements;
-	profile.height = feetInchesToCm(data.Height || data.height);
-	profile.weight = lbsToKg(data.Weight || data.weight);
-
-	profile.eyes = data.Eyes || data.eyes;
-	profile.hairColor = data.Hair || data.hair;
-
-	profile.avatar = data.thumb;
-
-	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
-
-	return profile;
-}
-
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.contents?.data) {
+		if (data?.props?.pageProps?.contents?.data) {
 			const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
 
 			return scenes;
@@ -86,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
 		return baseRelease;
 	}
 
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.content) {
+		if (data?.props?.pageProps?.content) {
 			return scrapeScene(data.props.pageProps.content, channel);
 		}
 
@@ -102,15 +81,38 @@ async function fetchScene(url, channel, baseRelease) {
 	return res.status;
 }
 
+function scrapeProfile(data, scenes, entity) {
+	const profile = {};
+
+	profile.entryId = data.id;
+	profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
+
+	profile.description = data.Bio || data.bio;
+	profile.birthPlace = data.Born || data.born;
+	profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
+
+	profile.measurements = data.Measurements || data.Measurements;
+	profile.height = feetInchesToCm(data.Height || data.height);
+	profile.weight = lbsToKg(data.Weight || data.weight);
+
+	profile.eyes = data.Eyes || data.eyes;
+	profile.hairColor = data.Hair || data.hair;
+
+	profile.avatar = data.thumb;
+
+	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
+
+	return profile;
+}
+
 async function fetchProfile({ slug }, entity) {
 	const url = `${entity.url}/models/${slug}`;
-	const res = await qu.get(url);
+	const res = await unprint.get(url);
 
 	if (res.ok) {
-		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = res.context.query.json('#__NEXT_DATA__');
-		const data = dataString && JSON.parse(dataString);
 
-		if (data.props?.pageProps?.model) {
+		if (data?.props?.pageProps?.model) {
 			return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
 		}
 

src/scrapers/rsghmedia.js

@@ -0,0 +1,158 @@
+'use strict';
+
+const unprint = require('unprint');
+const escapeRegexp = require('escape-string-regexp').default;
+
+const slugify = require('../utils/slugify');
+const capitalize = require('../utils/capitalize');
+
+function getDate(query, release) {
+	const date = query.date('.published', 'MMMM D, YYYY') || query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' });
+
+	if (date) {
+		return {
+			date,
+			datePrecision: 'day',
+		};
+	}
+
+	const poster = Array.isArray(release.poster)
+		? release.poster[0]
+		: release.poster;
+
+	if (poster) {
+		const [, year, month] = new URL(poster).pathname.match(/\/uploads\/(\d{4})\/(\d{2})/) || [];
+
+		if (year && month) {
+			return {
+				date: new Date(Date.UTC(year, month - 1, 1)),
+				datePrecision: 'month',
+			};
+		}
+	}
+
+	return {};
+}
+
+function scrapeAll(scenes) {
+	return scenes.map(({ query, element }) => {
+		const release = {};
+
+		release.url = query.url('.entry-title a, a.entry-featured-image-url');
+		release.entryId = new URL(release.url).pathname.match(/\/([\w-]+)/)[1];
+
+		const postId = unprint.query.number(element, null, { attribute: 'id' });
+
+		if (postId) {
+			release.attributes = { postId };
+		}
+
+		release.title = query.content('.entry-title a');
+		// description is cut off
+
+		release.tags = query.contents('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]'); // tag only seen on scene page, but just in case
+		release.poster = Array.from(new Set(query.sourceSet('.entry-featured-image-url img')));
+
+		const { date, datePrecision } = getDate(query, release);
+
+		release.date = date;
+		release.datePrecision = datePrecision;
+
+		return release;
+	}).toSorted((sceneA, sceneB) => {
+		// HTML is laid out in columns that does not align with chronology
+		if (sceneA.attributes?.postId && sceneB.attributes?.postId) {
+			return sceneB.attributes.postId - sceneA.attributes.postId;
+		}
+
+		if (sceneA.date && sceneB.date) {
+			return sceneB.date - sceneA.date;
+		}
+
+		return 0;
+	});
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.origin}/page/${page}`;
+	const res = await unprint.get(url, { selectAll: '.et_pb_blog_grid .et_pb_post' });
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel);
+	}
+
+	return res.status;
+}
+
+function scrapeScene({ query }, { url }) {
+	const release = {};
+
+	release.entryId = new URL(url).pathname.match(/\/([\w-]+)/)[1];
+
+	const postId = query.number('article[id*="post"]', { attribute: 'id' });
+
+	if (postId) {
+		release.attributes = { postId };
+	}
+
+	release.title = query.content('.entry-title');
+	release.description = query.content('.entry-content p');
+
+	release.actors = query.all('.st-post-tags a[href*="/tag"]').map((actorEl) => ({
+		name: unprint.query.content(actorEl),
+		url: unprint.query.url(actorEl, null),
+	}));
+
+	release.poster = query.img('meta[property="og:image"], meta[property="twitter:image"]', { attribute: 'content' })
+		|| query.sourceSet('.et_post_meta_wrapper img');
+
+	release.trailer = query.video('.entry-content source');
+
+	const { date, datePrecision } = getDate(query, release);
+
+	release.date = date;
+	release.datePrecision = datePrecision;
+
+	const tags = query.all('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]');
+	const starring = query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "Starring")]')
+		|| query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "***")]')
+		|| release.title?.split(/\s[-–]\s/).at(-1);
+
+	if (tags.length > 0) {
+		release.tags = tags.map((tagEl) => unprint.query.content(tagEl));
+	} else {
+		release.tags = query.content('article[id*="post"]', { attribute: 'class' })
+			?.match(/(category|tag)-[\w-]+/g)
+			?.map((tag) => slugify(tag.replace(/(category|tag)-/, ''), ' '))
+			|| [];
+	}
+
+	if (release.actors.length === 0 && release.tags.length > 0 && starring) {
+		// actors are always added as tags, sometimes they're clearly divided, sometimes they're mixed in with other tags
+		// the tail end of the description or title contains a *starring* section against which the tags can be matched
+		release.actors = release.tags
+			.map((tag) => {
+				if (tag === 'alex adams') {
+					// household male performer that is not usually mentioned in the starring section
+					return capitalize(tag);
+				}
+
+				// use description text instead of tag name for capitalization
+				const match = starring.match(new RegExp(`\\b${escapeRegexp(tag)}\\b`, 'i'))?.[0];
+
+				if (match) {
+					return match;
+				}
+
+				return null;
+			})
+			.filter(Boolean);
+	}
+
+	return release;
+}
+
+module.exports = {
+	fetchLatest,
+	scrapeScene,
+};

src/scrapers/snowvalley.js

@@ -47,11 +47,7 @@ function scrapeAll(scenes, tilesByEntryId, channel) {
 
 		release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
 
-		const posterBackground = query.style('.player');
+		release.poster = query.background('.player');
-
-		if (posterBackground?.background) {
-			release.poster = posterBackground.background.match(/url\((.*)\)/)?.[1]?.trim();
-		}
 
 		release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
 			unprint.query.url(linkEl, null),
@@ -121,8 +117,7 @@ function scrapeAllTiles(tiles, channel) {
 
 		release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
 
-		const posterBackground = query.style('.scene-img');
+		const posterUrl = query.background('.scene-img');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = [
@@ -199,8 +194,7 @@ function scrapeAllCospuri(scenes, channel) {
 
 		release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
 
-		const posterBackground = query.style('.scene-thumb');
+		const posterUrl = query.background('.scene-thumb');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = [
@@ -260,8 +254,7 @@ function scrapeAllFellatio(scenes, channel) {
 
 		release.tags = query.contents('.tour-data a[href*="tag/"]');
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = posterUrl;
@@ -302,8 +295,7 @@ function scrapeAllHandjob(scenes, _channel) {
 
 		release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = posterUrl;
@@ -348,8 +340,7 @@ function scrapeAllLegs(scenes, channel) {
 
 		release.tags = query.contents('a[href*="tag/"]');
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = posterUrl;
@@ -385,8 +376,7 @@ function scrapeAllFacefuck(scenes) {
 		release.description = query.content('.infotxt');
 		release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = posterUrl;
@@ -425,8 +415,7 @@ function scrapeAllTrans(scenes) {
 		release.duration = query.duration('.sample-info');
 		release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = [
@@ -436,7 +425,7 @@ function scrapeAllTrans(scenes) {
 			];
 		}
 
-		release.photos = curatePhotos(albumContext?.query.styles('.sample-lg, .sample-thumb').map((style) => style['background-image']?.match(/url\((.*)\)/)?.[1]));
+		release.photos = curatePhotos(albumContext?.query.backgrounds('.sample-lg, .sample-thumb'));
 		release.trailer = query.video();
 
 		release.entryId = entryIdFromMedia(release);
@@ -486,8 +475,7 @@ function scrapeAllLesbianTiles(scenes, channel) {
 
 		release.tags = query.contents('.content-tags a');
 
-		const posterBackground = query.style('.vidthumb');
+		const posterUrl = query.background('.vidthumb');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = [
@@ -522,8 +510,7 @@ function scrapeAllLesbian(scenes, channel, tiles) {
 		release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
 		release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
 
-		const posterBackground = query.style('.player');
+		const posterUrl = query.background('.player');
-		const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 		if (posterUrl) {
 			release.poster = posterUrl;
@@ -650,8 +637,7 @@ function scrapeSceneBuffet({ query }, { url, entity }) {
 
 	release.tags = query.contents('.tag-list a');
 
-	const posterBackground = query.style('.player');
+	const posterUrl = query.background('.player');
-	const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 	if (posterUrl) {
 		release.poster = [
@@ -696,8 +682,7 @@ function scrapeSceneCospuri({ query }, { url, entity }) {
 
 	release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
 
-	const posterBackground = query.style('.player');
+	const posterUrl = query.background('.player');
-	const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
 
 	if (posterUrl) {
 		release.poster = posterUrl;
@@ -788,7 +773,7 @@ function scrapeProfile({ query }, _channel, url) {
 
 	const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
 		|| query.img('.costume-bg', { attribute: 'data-img' })
-		|| query.style('.model-profile, #profile, .carousel-item')?.['background-image']?.match(/url\((.*)\)/)?.[1];
+		|| query.background('.model-profile, #profile, .carousel-item');
 
 	if (avatar) {
 		profile.avatar = Array.from(new Set([

src/scrapers/teamskeet.js

@@ -2,7 +2,6 @@
 
 const unprint = require('unprint');
 
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const { convert } = require('../utils/convert');
 
@@ -34,7 +33,8 @@ async function fetchTrailerUrl(videoId, entity) {
 		const token = res.data.token;
 
 		if (token) {
-			return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`;
+			// ffmpeg fails to read the DASH manifest over a single connection, HLS works
+			return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.m3u8?parentOrigin=${entity.origin}`;
 		}
 	}
 
@@ -47,17 +47,18 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
 	// release.entryId = scene.id; // legacy
 	release.entryId = scene.itemId;
 
-	release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
+	release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
 
 	release.title = scene.title;
 	release.description = scene.description;
 	release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
+	release.duration = scene.d;
 
 	release.actors = scene.models?.map((model) => ({
 		name: model.modelName || model.name || model.title,
 		avatar: model.img,
 		gender: model.gender,
-		url: `${channel.url}/models/${model.modelId || model.id}`,
+		url: `${channel.origin}/models/${model.modelId || model.id}`,
 	}));
 
 	if (scene.img) {
@@ -93,16 +94,16 @@ function scrapeAll(scenes, channel, parameters) {
 
 async function fetchLatest(channel, page = 1, { parameters }) {
 	// url: 'https://www.pervz.com/series/pervz-features',
-	const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
+	const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
 
 	const url = seriesId
 		? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
 		: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
 
-	const res = await http.get(url);
+	const res = await unprint.get(url);
 
-	if (res.ok) {
+	if (res.ok && res.data?.hits) {
-		return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
+		return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
 	}
 
 	return res.status;
@@ -114,27 +115,22 @@ async function fetchScene(url, channel, baseScene, { parameters, includeTrailers
 		return baseScene;
 	}
 
-	const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
+	const sceneSlug = new URL(url).pathname.match(/\/([\w-]+)\/?$/)?.[1];
 
-	const res = await unprint.get(url, {
+	if (!sceneSlug) {
-		parser: {
+		return null;
-			runScripts: 'dangerously',
+	}
-		},
-	});
 
-	if (res.ok) {
+	// scene page no longer embeds state, look up the scene in the elastic API by its slug instead
-		const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent;
+	// legacy URLs use the numeric item ID, which 404s on the site but can still be looked up
+	const isItemId = /^\d+$/.test(sceneSlug);
+	const res = await unprint.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(${isItemId ? `itemId:${sceneSlug}` : `id:"${sceneSlug}"`} AND type:video)&size=1`);
 
-		res.context.window.fetch = () => {}; // suppress fetch missing error
+	if (res.ok && res.data?.hits) {
+		const scene = res.data.hits.hits.find(({ _source: hit }) => (isItemId ? String(hit.itemId) : hit.id) === sceneSlug)?._source;
 
-		if (!videos) {
+		if (scene) {
-			return null;
+			return scrapeScene(scene, channel, parameters, includeTrailers);
-		}
-
-		const video = videos?.[sceneSlug] || Object.values(videos)[0];
-
-		if (video) {
-			return scrapeScene(video, channel, parameters, includeTrailers);
 		}
 
 		return null;

src/scrapers/template.js

@@ -31,7 +31,7 @@ function scrapeAll(scenes) {
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `${channel.url}/${page}`;
+	const url = `${channel.origin}/${page}`;
 	const res = await unprint.get(url, { selectAll: '.scene' });
 
 	if (res.ok) {

src/scrapers/topwebmodels.js

@@ -1,144 +0,0 @@
-'use strict';
-
-const mime = require('mime');
-
-const qu = require('../utils/qu');
-const http = require('../utils/http');
-const slugify = require('../utils/slugify');
-const { convert } = require('../utils/convert');
-
-function scrapeSceneX(scene) {
-	const release = {};
-
-	release.entryId = scene.id;
-	release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
-
-	release.title = scene.title;
-	release.description = scene.description;
-
-	release.duration = qu.durationToSeconds(scene.videos_duration);
-	release.date = new Date(scene.release_date);
-
-	release.actors = scene.models
-		.map((actor) => (/&/.test(actor.name)
-			? actor.name.split(/\s*&\s*/)
-			: {
-				name: actor.name,
-				gender: actor.gender || null,
-				avatar: actor.thumb,
-				url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
-			}))
-		.flat();
-
-	release.stars = scene.rating;
-	release.tags = scene.tags.map((tag) => tag.name);
-
-	if (mime.getType(scene.thumb) === 'image/gif') {
-		release.teaser = scene.thumb;
-	} else {
-		release.poster = scene.thumb;
-	}
-
-	release.channel = slugify(scene.sites[0]?.name, '');
-
-	return release;
-}
-
-function scrapeAll(scenes) {
-	return scenes.map(scrapeSceneX);
-}
-
-async function scrapeProfile(actor, options) {
-	const profile = {};
-
-	profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
-	profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
-	profile.gender = actor.attributes.gender || null;
-
-	profile.height = convert(actor.attributes.height.value, 'cm');
-	profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
-
-	const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
-
-	profile.bust = Number(bust);
-	profile.cup = cup;
-	profile.waist = Number(waist);
-	profile.hip = Number(hip);
-
-	profile.ethnicity = actor.attributes.ethnicity.value;
-	profile.birthPlace = actor.attributes.born.value;
-
-	profile.eyes = actor.attributes.eyes.value;
-	profile.hairColor = actor.attributes.hair.value.split('/')[0];
-
-	profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
-	profile.avatar = actor.thumb;
-
-	if (options.includeActorScenes) {
-		const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
-
-		if (res.ok) {
-			profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
-		}
-	}
-
-	return profile;
-}
-
-async function fetchLatest(channel, page) {
-	const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
-		headers: {
-			Referer: 'https://tour.topwebmodels.com',
-			'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
-			'x-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (res.ok) {
-		return scrapeAll(res.body.videos.items);
-	}
-
-	return res.status;
-}
-
-async function fetchScene(url) {
-	const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
-
-	if (res.ok) {
-		return {
-			...scrapeSceneX(res.window.__DATA__.data.video),
-			...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
-			poster: res.window.__DATA__.data.file_poster,
-		};
-	}
-
-	return res.status;
-}
-
-async function fetchProfile(baseActor, entity, options) {
-	const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
-		headers: {
-			Referer: 'https://tour.topwebmodels.com',
-			'api-key': entity.parameters?.apiKey,
-			'x-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (!searchRes.ok) {
-		return searchRes.status;
-	}
-
-	const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
-
-	if (actor) {
-		return scrapeProfile(actor, options);
-	}
-
-	return null;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchScene,
-	fetchProfile,
-};

src/scrapers/traxxx.js

@@ -241,8 +241,8 @@ function generateActors(release) {
 		: Math.floor(Math.random() * 3) + 2;
 
 	return Array.from({ length }, () => ({
-		name: faker.name
+		name: faker.person
-			.findName()
+			.fullName()
 			.split(' ')
 			.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
 			.join(' '),

src/scrapers/vixen.js

@@ -74,7 +74,9 @@ function scrapeAll(scenes, channel) {
 
 async function fetchLatest(site, page = 1) {
 	const url = `${site.url}/videos?page=${page}`;
-	const res = await unprint.get(url);
+	const res = await unprint.get(url, {
+		interface: 'request',
+	});
 
 	if (res.ok) {
 		const data = res.context.query.json('#__NEXT_DATA__');

src/stashes.js

@@ -1,312 +0,0 @@
-'use strict';
-
-const config = require('config');
-
-const knex = require('./knex');
-const { indexApi } = require('./manticore');
-const { HttpError } = require('./errors');
-const slugify = require('./utils/slugify');
-const logger = require('./logger')(__filename);
-
-let lastActorsViewRefresh = 0;
-
-function curateStash(stash) {
-	if (!stash) {
-		return null;
-	}
-
-	const curatedStash = {
-		id: stash.id,
-		name: stash.name,
-		slug: stash.slug,
-		primary: stash.primary,
-	};
-
-	return curatedStash;
-}
-
-function curateStashEntry(stash, user) {
-	const curatedStashEntry = {
-		user_id: user.id,
-		name: stash.name,
-		slug: slugify(stash.name),
-		public: false,
-	};
-
-	return curatedStashEntry;
-}
-
-async function fetchStash(stashId, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	const stash = await knex('stashes')
-		.where({
-			id: stashId,
-			user_id: sessionUser.id,
-		})
-		.first();
-
-	if (!stash) {
-		throw new HttpError('You are not authorized to access this stash', 403);
-	}
-
-	return curateStash(stash);
-}
-
-async function fetchStashes(domain, itemId, sessionUser) {
-	const stashes = await knex(`stashes_${domain}s`)
-		.select('stashes.*')
-		.where({
-			[`${domain}_id`]: itemId,
-			user_id: sessionUser.id,
-		})
-		.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
-
-	return stashes.map((stash) => curateStash(stash));
-}
-
-async function createStash(newStash, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	try {
-		const stash = await knex('stashes')
-			.insert(curateStashEntry(newStash, sessionUser))
-			.returning('*');
-
-		return curateStash(stash);
-	} catch (error) {
-		if (error.routine === '_bt_check_unique') {
-			throw new HttpError('Stash name should be unique', 409);
-		}
-
-		throw error;
-	}
-}
-
-async function updateStash(stashId, newStash, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	const stash = await knex('stashes')
-		.where({
-			id: stashId,
-			user_id: sessionUser.id,
-		})
-		.update(newStash)
-		.returning('*');
-
-	if (!stash) {
-		throw new HttpError('You are not authorized to modify this stash', 403);
-	}
-
-	return curateStash(stash);
-}
-
-async function removeStash(stashId, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	const removed = await knex('stashes')
-		.where({
-			id: stashId,
-			user_id: sessionUser.id,
-			primary: false,
-		})
-		.delete();
-
-	if (removed === 0) {
-		throw new HttpError('Unable to remove this stash', 400);
-	}
-}
-
-async function refreshActorsView() {
-	if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
-		// don't refresh actors view more than once an hour
-		lastActorsViewRefresh = new Date();
-
-		logger.debug('Refreshing actors view');
-
-		return knex.schema.refreshMaterializedView('actors_meta');
-	}
-
-	logger.silly('Skipping actors view refresh');
-
-	return false;
-}
-
-async function stashActor(actorId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
-
-	const [stashed] = await knex('stashes_actors')
-		.insert({
-			stash_id: stash.id,
-			actor_id: actorId,
-		})
-		.returning(['id', 'created_at']);
-
-	await indexApi.replace({
-		index: 'actors_stashed',
-		id: stashed.id,
-		doc: {
-			actor_id: actorId,
-			user_id: sessionUser.id,
-			stash_id: stashId,
-			created_at: Math.round(stashed.created_at.getTime() / 1000),
-		},
-	});
-
-	refreshActorsView();
-
-	return fetchStashes('actor', actorId, sessionUser);
-}
-
-async function stashScene(sceneId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
-
-	const [stashed] = await knex('stashes_scenes')
-		.insert({
-			stash_id: stash.id,
-			scene_id: sceneId,
-		})
-		.returning(['id', 'created_at']);
-
-	await indexApi.replace({
-		index: 'scenes_stashed',
-		id: stashed.id,
-		doc: {
-			// ...doc.replace.doc,
-			scene_id: sceneId,
-			user_id: sessionUser.id,
-			stash_id: stashId,
-			created_at: Math.round(stashed.created_at.getTime() / 1000),
-		},
-	});
-
-	return fetchStashes('scene', sceneId, sessionUser);
-}
-
-async function stashMovie(movieId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
-
-	const [stashed] = await knex('stashes_movies')
-		.insert({
-			stash_id: stash.id,
-			movie_id: movieId,
-		})
-		.returning(['id', 'created_at']);
-
-	await indexApi.replace({
-		index: 'movies_stashed',
-		id: stashed.id,
-		doc: {
-			movie_id: movieId,
-			user_id: sessionUser.id,
-			stash_id: stashId,
-			created_at: Math.round(stashed.created_at.getTime() / 1000),
-		},
-	});
-
-	return fetchStashes('movie', movieId, sessionUser);
-}
-
-async function unstashActor(actorId, stashId, sessionUser) {
-	await knex
-		.from('stashes_actors AS deletable')
-		.where('deletable.actor_id', actorId)
-		.where('deletable.stash_id', stashId)
-		.whereExists(knex('stashes_actors') // verify user owns this stash, complimentary to row-level security
-			.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
-			.where('stashes_actors.stash_id', knex.raw('deletable.stash_id'))
-			.where('stashes.user_id', sessionUser.id))
-		.delete();
-
-	await indexApi.callDelete({
-		index: 'actors_stashed',
-		query: {
-			bool: {
-				must: [
-					{ equals: { actor_id: actorId } },
-					{ equals: { stash_id: stashId } },
-					{ equals: { user_id: sessionUser.id } },
-				],
-			},
-		},
-	});
-
-	refreshActorsView();
-
-	return fetchStashes('actor', actorId, sessionUser);
-}
-
-async function unstashScene(sceneId, stashId, sessionUser) {
-	await knex
-		.from('stashes_scenes AS deletable')
-		.where('deletable.scene_id', sceneId)
-		.where('deletable.stash_id', stashId)
-		.whereExists(knex('stashes_scenes') // verify user owns this stash, complimentary to row-level security
-			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
-			.where('stashes_scenes.stash_id', knex.raw('deletable.stash_id'))
-			.where('stashes.user_id', sessionUser.id))
-		.delete();
-
-	await indexApi.callDelete({
-		index: 'scenes_stashed',
-		query: {
-			bool: {
-				must: [
-					{ equals: { scene_id: sceneId } },
-					{ equals: { stash_id: stashId } },
-					{ equals: { user_id: sessionUser.id } },
-				],
-			},
-		},
-	});
-
-	return fetchStashes('scene', sceneId, sessionUser);
-}
-
-async function unstashMovie(movieId, stashId, sessionUser) {
-	await knex
-		.from('stashes_movies AS deletable')
-		.where('deletable.movie_id', movieId)
-		.where('deletable.stash_id', stashId)
-		.whereExists(knex('stashes_movies') // verify user owns this stash, complimentary to row-level security
-			.leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id')
-			.where('stashes_movies.stash_id', knex.raw('deletable.stash_id'))
-			.where('stashes.user_id', sessionUser.id))
-		.delete();
-
-	await indexApi.callDelete({
-		index: 'movies_stashed',
-		query: {
-			bool: {
-				must: [
-					{ equals: { movie_id: movieId } },
-					{ equals: { stash_id: stashId } },
-					{ equals: { user_id: sessionUser.id } },
-				],
-			},
-		},
-	});
-
-	return fetchStashes('movie', movieId, sessionUser);
-}
-
-module.exports = {
-	createStash,
-	curateStash,
-	removeStash,
-	stashActor,
-	stashScene,
-	stashMovie,
-	unstashScene,
-	unstashActor,
-	unstashMovie,
-	updateStash,
-};

src/store-releases.js

@@ -2,7 +2,7 @@
 
 const config = require('config');
 const Promise = require('bluebird');
-const { decode } = require('html-entities');
+const { decode: decodeEntities } = require('html-entities');
 const unprint = require('unprint');
 
 const argv = require('./argv');
@@ -12,13 +12,10 @@ const slugify = require('./utils/slugify');
 const bulkInsert = require('./utils/bulk-insert');
 const chunk = require('./utils/chunk');
 const redis = require('./redis');
-const { formatDate } = require('./utils/qu');
 const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
 const { associateReleaseTags } = require('./tags');
 const { curateEntity } = require('./entities');
 const { associateReleaseMedia } = require('./media');
-const { updateSceneSearch, updateMovieSearch } = require('./update-search');
-const { notify } = require('./alerts');
 
 const geoCommon = import('../common/geo.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
 
@@ -31,10 +28,15 @@ const commonContext = {
 	unprint,
 };
 
+// newer html-entities throws on non-strings, pass e.g. numeric titles through like 2.4 did
+function decode(text) {
+	return typeof text === 'string' ? decodeEntities(text) : text;
+}
+
 async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
 	const slugBase = release.title
 		|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
-		|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
+		|| (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`)
 		|| null;
 
 	const slug = slugify(slugBase, '-', {
@@ -345,7 +347,8 @@ async function associateMovieScenes(movies, movieScenes) {
 
 	await bulkInsert('movies_scenes', associations, false);
 
-	await updateSceneSearch(movieScenes.map((scene) => scene.id));
+	// movieScenes includes scenes that are not associated to any movies, don't update those (again)
+	// await updateSceneSearch(associations.map((association) => association.scene_id));
 }
 
 async function associateSerieScenes(series, serieScenes) {
@@ -392,7 +395,7 @@ async function storeMovies(movies, useBatchId) {
 	const moviesWithId = attachReleaseIds(movies, storedMovies);
 
 	await associateReleaseTags(moviesWithId, 'movie');
-	await updateMovieSearch(moviesWithId.map((movie) => movie.id));
+	// await updateMovieSearch(moviesWithId.map((movie) => movie.id));
 
 	await associateReleaseMedia(moviesWithId, 'movie');
 
@@ -420,7 +423,8 @@ async function storeSeries(series, useBatchId) {
 	const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
 	const seriesWithId = attachReleaseIds(series, storedSeries);
 
-	await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
+	// TODO: the movie search function doesn't accept a second parameter, and we don't currently have a series search index at all; re-evaluate what actually needs to happen here
+	// await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
 	await associateReleaseMedia(seriesWithId, 'serie');
 
 	return seriesWithId;
@@ -490,7 +494,7 @@ async function storeScenes(releases, useBatchId) {
 		await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
 	}
 
-	await updateSceneSearch(releasesWithId.map((release) => release.id));
+	// await updateSceneSearch(releasesWithId.map((release) => release.id));
 
 	// media is more error-prone, associate separately
 	await associateReleaseMedia(releasesWithId);
@@ -503,8 +507,6 @@ async function storeScenes(releases, useBatchId) {
 
 	logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated} releases`);
 
-	await notify(releasesWithId);
-
 	return releasesWithId;
 }
 
@@ -512,6 +514,4 @@ module.exports = {
 	associateMovieScenes,
 	storeScenes,
 	storeMovies,
-	updateSceneSearch,
-	updateMovieSearch,
 };

src/tags.js

@@ -79,13 +79,14 @@ function withRelations(queryBuilder, withMedia) {
 }
 
 async function matchTags(rawTags) {
-	const tags = rawTags
+	const tags = Array.from(new Set(rawTags
-		.map((tag) => tag?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
+		.map((tag) => (typeof tag === 'string' ? tag : null)?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
-		.filter(Boolean);
+		.filter(Boolean)));
 
+	// array binding, large batches can easily exceed the bindings limit otherwise
 	const tagEntries = await knex('tags')
 		.select('tags.id', 'tags.name', 'tags.alias_for', 'tags.implied_tag_ids')
-		.whereIn(knex.raw('lower(tags.name)'), tags);
+		.whereRaw('lower(tags.name) = ANY(:tags)', { tags });
 
 	const tagIdsBySlug = tagEntries
 		.reduce((acc, tag) => ({

src/tools/analvids.js

@@ -1,84 +0,0 @@
-'use strict';
-
-const fs = require('fs').promises;
-const bhttp = require('bhttp');
-const slugify = require('../utils/slugify');
-const { studios: oldStudios } = require('../../seeds/03_studios');
-
-/*
-const slugMap = {
-	thay_ksada: 'thayksada2',
-};
-*/
-
-async function init() {
-	const res = await bhttp.get('https://pornbox.com/studio/list');
-
-	if (!res.statusCode === 200) {
-		throw new Error(`Failed to retrieve studios (${res.statusCode}): ${res.body.toString()}`);
-	}
-
-	const newStudios = res.body.map((studio) => {
-		const slug = slugify(studio.name, '');
-		const oldStudio = oldStudios.find((oldStudioX) => oldStudioX.slug === slug);
-
-		const tags = oldStudio?.tags || [];
-
-		const newStudio = {
-			name: studio.name,
-			slug,
-			originalSlug: slugify(studio.alias, ''),
-			url: `https://www.analvids.com/studios/${studio.alias}`,
-			parent: 'analvids',
-		};
-
-		if (oldStudio?.alias) {
-			newStudio.alias = oldStudio.alias;
-		}
-
-		if (studio.description && studio.description.toLowerCase() !== studio.name.toLowerCase()) {
-			newStudio.description = studio.description;
-		}
-
-		if (studio.directions?.includes('gay')) {
-			tags.push('gay');
-		}
-
-		if (studio.directions?.includes('lesbian')) {
-			tags.push('lesbian');
-		}
-
-		if (studio.directions?.includes('trans')) {
-			tags.push('transsexual');
-		}
-
-		if (tags.length > 0) {
-			newStudio.tags = Array.from(new Set(tags));
-		}
-
-		return newStudio;
-	});
-
-	await fs.writeFile('./analvids-studios.json', JSON.stringify(newStudios, null, 4));
-
-	newStudios.reduce((acc, studio) => {
-		const existingStudio = acc.get(studio.slug);
-
-		if (existingStudio) {
-			if (!acc.has(studio.originalSlug)) {
-				console.log('REWRITE', studio.slug, studio.originalSlug);
-				studio.slug = studio.originalSlug; // eslint-disable-line
-			}
-
-			console.log('slug already exists!', existingStudio, studio, studio.originalSlug, studio.slug);
-		}
-
-		acc.set(studio.slug, studio);
-
-		return acc;
-	}, new Map());
-
-	// console.log(newStudios);
-}
-
-init();

src/tools/dogfart.js

@@ -1,76 +0,0 @@
-'use strict';
-
-const fs = require('fs').promises;
-
-const Promise = require('bluebird');
-
-const qu = require('../utils/qu');
-
-const qualities = {
-	sm: 360,
-	med: 480,
-	big: 720,
-};
-
-/*
-async function scrape() {
-	const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
-		const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
-
-		if (res.ok) {
-			return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
-		}
-
-		return [];
-	}, { concurrency: 1 });
-
-	const urls = urlsByPage.flat();
-
-	await fs.writeFile('./dogfart-links', urls.join('\n'));
-
-	console.log(`Saved ${urls.length} URLs to file`);
-}
-
-async function compare() {
-	const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
-	const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
-
-	const newLinks = newLinksFile.split('\n').filter(Boolean);
-	const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
-
-	const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
-
-	await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
-
-	console.log(getLinks);
-}
-*/
-
-async function scrapeMembers() {
-	const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
-		const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
-
-		if (res.ok) {
-			return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
-				url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
-				actors: query.contents('a[href*="model.php"]'),
-				trailer: query.urls('.trailer-link a').map((url) => ({
-					src: url,
-					quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
-				})),
-			}));
-		}
-
-		return [];
-	}, { concurrency: 1 });
-
-	const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
-
-	console.log(urls);
-
-	await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
-
-	console.log(`Saved ${urls.length} URLs to file`);
-}
-
-scrapeMembers();

src/tools/kink.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const bhttp = require('bhttp');
-
-async function init() {
-	const res = await bhttp.head('https://www.kink.com/consent', {
-		headers: {
-			Host: 'www.kink.com',
-			'User-Agent': 'HTTPie/2.6.0',
-			Accept: '*/*',
-			Connection: 'keep-alive',
-		},
-	});
-
-	console.log(res.body.toString(), res.headers, res.statusCode);
-}
-
-init();

src/tools/realitykings.js

@@ -1,40 +0,0 @@
-'use strict';
-
-const fetch = require('node-fetch');
-const express = require('express');
-
-async function init() {
-	const res = await fetch('https://www.realitykings.com/scenes?site=45', {
-		method: 'HEAD',
-		headers: {
-			'user-agent': 'HTTPie/2.6.0',
-			'accept-encoding': 'gzip, deflate, br',
-			accept: '*/*',
-			connection: 'keep-alive',
-		},
-	});
-
-	console.log(res.status, res.headers);
-
-	const app = express();
-
-	app.get('/', (appReq, appRes) => {
-		console.log(appReq.headers);
-		appRes.status(204).send();
-	});
-
-	app.listen(8000, () => {
-		console.log('Listening on port 8000');
-
-		fetch('http://127.0.0.1:8000', {
-			headers: {
-				'user-agent': 'HTTPie/2.6.0',
-				'accept-encoding': 'gzip, deflate, br',
-				accept: '*/*',
-				connection: 'keep-alive',
-			},
-		});
-	});
-}
-
-init();

src/tools/server.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const express = require('express');
-
-const app = express();
-
-app.get('/', (req, res) => {
-	console.log(req.httpVersion);
-	console.log(req.headers);
-	console.log(req.body);
-
-	res.status(204).send();
-});
-
-app.listen(5555, () => {
-	console.log('Server started');
-});

src/tools/stashdb-hashes.js

@@ -2,7 +2,7 @@
 
 // const util = require('util');
 const unprint = require('unprint');
-const args = require('yargs').argv;
+const args = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
 
 const knex = require('../knex');
 const bulkInsert = require('../utils/bulk-insert');

src/update-search.js

@@ -10,7 +10,7 @@ async function syncWeb(domain, ids) {
 		return;
 	}
 
-	await knex('sync').insert({ domain, item_ids: ids });
+	await knex('sync').insert({ domain, item_ids: Array.from(new Set(ids)) });
 
 	if (config.webApi.enabled) {
 		await unprint.post(`${config.webApi.address}/sync`, null, {

src/users.js

@@ -1,56 +0,0 @@
-'use strict';
-
-const knex = require('./knex');
-const { curateStash } = require('./stashes');
-
-function curateUser(user) {
-	if (!user) {
-		return null;
-	}
-
-	const ability = [...(user.role_abilities || []), ...(user.abilities || [])];
-
-	const curatedUser = {
-		id: user.id,
-		username: user.username,
-		email: user.email,
-		emailVerified: user.email_verified,
-		identityVerified: user.identity_verified,
-		ability,
-		createdAt: user.created_at,
-		stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
-	};
-
-	return curatedUser;
-}
-
-async function fetchUser(userId, raw) {
-	const user = await knex('users')
-		.select(knex.raw('users.*, users_roles.abilities as role_abilities, COALESCE(json_agg(stashes ORDER BY stashes.created_at) FILTER (WHERE stashes.id IS NOT NULL), \'[]\') as stashes'))
-		.modify((builder) => {
-			if (typeof userId === 'number') {
-				builder.where('users.id', userId);
-			}
-
-			if (typeof userId === 'string') {
-				builder
-					.where('users.username', userId)
-					.orWhere('users.email', userId);
-			}
-		})
-		.leftJoin('users_roles', 'users_roles.role', 'users.role')
-		.leftJoin('stashes', 'stashes.user_id', 'users.id')
-		.groupBy('users.id', 'users_roles.role')
-		.first();
-
-	if (raw) {
-		return user;
-	}
-
-	return curateUser(user);
-}
-
-module.exports = {
-	curateUser,
-	fetchUser,
-};

src/utils/buffer.js

@@ -1,118 +0,0 @@
-'use strict';
-
-const bhttp = require('bhttp');
-const Promise = require('bluebird');
-const fsPromises = require('fs').promises;
-const fs = require('fs');
-const { PassThrough } = require('stream');
-const blake2 = require('blake2');
-const argv = require('yargs').argv;
-
-const file = 'https://speed.hetzner.de/100MB.bin';
-// const file = 'https://speed.hetzner.de/1GB.bin';
-// const file = 'https://speed.hetzner.de/10GB.bin';
-
-function getMemoryUsage() {
-	return process.memoryUsage().rss / (10 ** 6);
-}
-
-const stats = {
-	peakMemoryUsage: getMemoryUsage(),
-	done: false,
-	downloads: {},
-};
-
-function render() {
-	const downloads = Object.entries(stats.downloads);
-
-	process.stdout.clearScreenDown();
-
-	process.stdout.write(`peak memory: ${stats.peakMemoryUsage.toFixed(2)} MB\n`);
-
-	downloads.forEach(([download, progress]) => {
-		process.stdout.write(`${download}: ${progress}${typeof progress === 'string' ? '' : '%'}\n`);
-	});
-
-	process.stdout.moveCursor(0, -(downloads.length + 1));
-	process.stdout.cursorTo(0);
-
-	if (downloads.length === 0 || !downloads.every(([_label, download]) => typeof download === 'string')) {
-		setTimeout(() => render(), 1000);
-		return;
-	}
-
-	process.stdout.moveCursor(0, downloads.length + 1);
-}
-
-function setProgress(label, completedBytes, totalBytes, hash) {
-	const memory = getMemoryUsage();
-
-	stats.peakMemoryUsage = Math.max(memory, stats.peakMemoryUsage);
-	stats.downloads[label] = hash || Math.round((completedBytes / totalBytes) * 100);
-}
-
-async function buffered(label) {
-	const hash = new blake2.Hash('blake2b');
-
-	const imageRes = await bhttp.get(file, {
-		onDownloadProgress(completedBytes, totalBytes) {
-			setProgress(label, completedBytes, totalBytes);
-		},
-	});
-
-	hash.update(imageRes.body);
-	setProgress(label, null, null, hash.digest('hex'));
-
-	await fsPromises.writeFile(`/mnt/stor/Pictures/traxxx/temp/buffered-${label}.bin`, imageRes.body);
-}
-
-async function streamed(label) {
-	const hash = new blake2.Hash('blake2b');
-	hash.setEncoding('hex');
-
-	const hashStream = new PassThrough();
-	const targetStream = fs.createWriteStream(`/mnt/stor/Pictures/traxxx/temp/streamed-${label}.bin`);
-
-	const imageRes = await bhttp.get(file, {
-		stream: true,
-	});
-
-	const stream = imageRes
-		.pipe(hashStream)
-		.pipe(targetStream);
-
-	imageRes.on('progress', (completedBytes, totalBytes) => {
-		setProgress(label, completedBytes, totalBytes);
-	});
-
-	hashStream.on('data', (chunk) => {
-		hash.write(chunk);
-	});
-
-	stream.on('finish', () => {
-		hash.end();
-		setProgress(label, null, null, hash.read());
-	});
-}
-
-async function init() {
-	const n = argv.n || 1;
-
-	if (argv._.includes('stream')) {
-		console.log('using streams');
-		render();
-
-		await Promise.map(Array.from({ length: n }), async (value, index) => streamed(index + 1));
-
-		return;
-	}
-
-	if (argv._.includes('buffer')) {
-		console.log('using buffers');
-		render();
-
-		await Promise.map(Array.from({ length: n }), async (value, index) => buffered(index + 1));
-	}
-}
-
-init();

src/utils/cf.js

@@ -1,153 +0,0 @@
-'use strict';
-
-const arrayEqual = require('array-equal');
-
-/* eslint-disable */
-function evaluateBranch(tree, modifiers) {
-	const result = tree.map((expression) => {
-		if (expression.type === 'group') {
-			return evaluateBranch(expression.values, expression.modifiers);
-		} if (expression.modifiers.length === 0) {
-			return ''; // This is a trigger to stringify the previous values
-		} if (arrayEqual(['plus'], expression.modifiers)) {
-			return 0;
-		} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
-			return true;
-		} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
-			return true;
-		} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
-			return true;
-		} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
-			return 1;
-		}
-		throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
-	}).reduce((combined, value) => {
-		if (value === '') {
-			return combined.toString();
-		}
-		if (value === true) {
-			value = 1;
-		}
-
-		if (typeof combined === 'string') {
-			return combined + value.toString();
-		}
-		return combined + value;
-	}, 0);
-
-	if (modifiers == null) {
-		return result;
-	}
-	if (arrayEqual(['plus'], modifiers)) {
-		return parseInt(result);
-	}
-	return result;
-}
-
-function evaluate(tree) {
-	return evaluateBranch(tree);
-}
-
-function parse(string) {
-	const length = string.length;
-	let byte;
-
-	let stateFinishedItem = false;
-	const modifierStack = [[]];
-	const itemStack = [[]];
-	let currentStack = itemStack[0];
-	let currentModifiers = modifierStack[0];
-	let stackLevel = 0;
-
-	for (let pos = 0; pos < length; pos++) {
-		byte = string[pos];
-
-		switch (byte) {
-		case '+':
-			if (pos === 0 || stateFinishedItem === false) {
-				// Modifier / number-cast
-				currentModifiers.push('plus');
-				stateFinishedItem = false;
-			} else {
-				// Addition, we don't need to do anything here
-			}
-			break;
-		case '!':
-			stateFinishedItem = false;
-			currentModifiers.push('negate');
-			break;
-		case '(':
-			stateFinishedItem = false;
-			stackLevel++;
-			itemStack[stackLevel] = currentStack = [];
-			modifierStack[stackLevel] = currentModifiers = [];
-			break;
-		case ')':
-			if (stackLevel === 0) {
-				throw new Error('Encountered ) without matching (');
-			}
-
-			stackLevel--;
-			stateFinishedItem = true;
-
-			currentStack = itemStack[stackLevel];
-			currentStack.push({
-				type: 'group',
-				values: itemStack[stackLevel + 1],
-				modifiers: modifierStack[stackLevel],
-			});
-
-			currentModifiers = modifierStack[stackLevel] = [];
-			break;
-		case '[':
-			if (string[pos + 1] === ']') {
-				// Reached the brackets; end of the modifier sequence
-				currentStack.push({
-					type: 'brackets',
-					modifiers: currentModifiers,
-				});
-
-				currentModifiers = [];
-				pos += 1; // Skip over the closing bracket
-				stateFinishedItem = true;
-			} else {
-				throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
-			}
-		}
-	}
-
-	return itemStack[0];
-}
-
-function parseExpression(string) {
-	return evaluate(parse(string));
-}
-
-function findAll(regex, target) {
-	const results = []; let
-		match;
-
-	while (match = regex.exec(target)) {
-		results.push(match);
-	}
-
-	return results;
-}
-
-function cf(testcase) {
-	let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
-
-	const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
-		operation: match[1],
-		expression: match[2],
-	})).map(({ operation, expression }) => ({
-		operation,
-		expression: parseExpression(expression),
-	}));
-
-	initialExpression = parseExpression(initialExpression);
-
-	return { parent, child, initialExpression, modifyingExpressions };
-};
-
-module.exports = cf;

src/utils/convert.js

@@ -82,7 +82,7 @@ function convertApi(rawInput, fromOrTo, to) {
 		return null;
 	}
 
-	const input = decode(rawInput); // remove html entities, e.g. 5' 8" for 5' 8"
+	const input = typeof rawInput === 'string' ? decode(rawInput) : rawInput; // remove html entities, e.g. 5' 8" for 5' 8"
 
 	try {
 		if (typeof input === 'string' && to === undefined) {

src/utils/http-windows.js

@@ -1,3 +0,0 @@
-'use strict';
-
-module.exports = new Map();

src/utils/http.js

@@ -1,511 +0,0 @@
-'use strict';
-
-const config = require('config');
-const Promise = require('bluebird');
-const bhttp = require('bhttp');
-const undici = require('undici');
-const fs = require('fs').promises;
-// const util = require('util');
-// const stream = require('stream');
-const { pipeline } = require('stream/promises');
-const Bottleneck = require('bottleneck');
-const { JSDOM, toughCookie } = require('jsdom');
-const puppeteer = require('puppeteer-extra');
-const StealthPlugin = require('puppeteer-extra-plugin-stealth');
-
-const windows = require('./http-windows');
-
-const logger = require('../logger')(__filename);
-const virtualConsole = require('./virtual-console')(__filename);
-const argv = require('../argv');
-
-// const pipeline = util.promisify(stream.pipeline);
-
-const limiters = {
-	bypass: new Bottleneck({
-		minTime: 1000,
-		maxConcurrent: 1,
-		timeout: 60000,
-	}),
-};
-
-const bypassSessions = new Map();
-
-const browsers = new Map();
-
-Promise.config({
-	cancellation: true,
-});
-
-puppeteer.use(StealthPlugin());
-
-const defaultOptions = {
-	timeout: argv.requestTimeout,
-	encodeJSON: true,
-	parse: false,
-	headers: {
-		'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
-	},
-};
-
-function useProxy(url) {
-	if (!config.proxy.enable) {
-		return false;
-	}
-
-	const { hostname } = new URL(url);
-
-	return config.proxy.hostnames.includes(hostname);
-}
-
-function useBrowserBypass(url, options) {
-	if (!config.bypass.browser.enable) {
-		return null;
-	}
-
-	const { hostname } = new URL(url);
-
-	if (options.bypassBrowser === 'shared') {
-		return true;
-	}
-
-	if (config.bypass.browser.hostnames.includes(hostname)) {
-		return true;
-	}
-
-	return false;
-}
-
-function useCloudflareBypass(url, options) {
-	if (!config.bypass.cloudflare.enable) {
-		return null;
-	}
-
-	const { hostname } = new URL(url);
-
-	if (options.bypassCloudflare === 'shared') {
-		return 'shared';
-	}
-
-	if (options.bypassCloudflare === 'independent') {
-		return hostname;
-	}
-
-	if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
-		return 'shared';
-	}
-
-	if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
-		return hostname;
-	}
-
-	return null;
-}
-
-const propMap = {
-	media: {
-		interval: 'mediaInterval',
-		concurrency: 'mediaConcurrency',
-	},
-};
-
-function getLimiterValue(prop, options, hostname) {
-	const mappedProp = propMap[options.limits]?.[prop] || prop;
-
-	if (typeof argv[mappedProp] !== 'undefined') {
-		return argv[mappedProp];
-	}
-
-	if (typeof options[prop] !== 'undefined') {
-		return options[prop];
-	}
-
-	if (config.limits[hostname]?.enable !== false && typeof config.limits[hostname]?.[prop] !== 'undefined') {
-		return config.limits[hostname][prop];
-	}
-
-	return config.limits[options.limits || 'default'][prop];
-}
-
-function getLimiter(options = {}, url) {
-	const { hostname } = new URL(url);
-
-	const interval = getLimiterValue('interval', options, hostname);
-	const concurrency = getLimiterValue('concurrency', options, hostname);
-
-	if (!limiters[interval]?.[concurrency]) {
-		limiters[interval] = limiters[interval] || {};
-
-		limiters[interval][concurrency] = new Bottleneck({
-			minTime: interval,
-			maxConcurrent: concurrency,
-			timeout: (options.timeout || defaultOptions.timeout) + 10000, // timeout 10 seconds after bhttp should
-		});
-	}
-
-	return limiters[interval][concurrency];
-}
-
-function extractJson(body, headers) {
-	if (headers['content-type'].includes('application/json')) {
-		const { document } = new JSDOM(body, { virtualConsole }).window;
-		const dataString = document.querySelector('body > pre')?.textContent;
-
-		if (dataString) {
-			const data = JSON.parse(dataString);
-
-			return data;
-		}
-	}
-
-	return body;
-}
-
-async function getBrowserSession(identifier, options = {}) {
-	return limiters.bypass.schedule(async () => {
-		const browserIdentifier = options.useGlobalBrowser === false ? identifier : null;
-
-		if (!browsers.has(browserIdentifier)) {
-			const newBrowser = await puppeteer.launch({
-				headless: typeof options.headless === 'undefined' ? 'new' : options.headless,
-				args: [
-					...(options.useProxy && config.proxy.enabled ? [`--proxy-server=${config.proxy.host}:${config.proxy.port}`] : []),
-				],
-				// headless: false,
-			});
-
-			browsers.set(browserIdentifier, newBrowser);
-
-			logger.info('Initialized puppeteer browser');
-		}
-
-		const browser = browsers.get(browserIdentifier);
-		const tab = await browser.newPage();
-
-		logger.verbose(`Opened puppeteer tab${identifier ? ` for ${identifier}` : ''}`);
-
-		return { browser, tab };
-	});
-}
-
-async function bypassBrowserRequest(url, options) {
-	const { tab } = await getBrowserSession(new URL(url).hostname, options);
-
-	const res = await tab.goto(url);
-
-	if (options.bypass?.delay) {
-		await Promise.delay(options.bypass.delay);
-	}
-
-	if (typeof options.bypass?.evaluate === 'function') {
-		await tab.evaluate(options.bypass.evaluate, options.bypass);
-	}
-
-	const rawBody = await tab.content();
-
-	const headers = res.headers();
-	const body = extractJson(rawBody, headers);
-
-	const statusCode = res.status();
-
-	if (!statusCode === 200) {
-		throw new Error(`Puppeteer bypass failed for ${url} (${statusCode}): ${body?.message}`);
-	}
-
-	return {
-		body,
-		statusCode,
-		headers,
-	};
-}
-
-async function getBypassSession(url, hostname) {
-	if (bypassSessions.has(hostname)) {
-		return bypassSessions.get(hostname);
-	}
-
-	const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
-		cmd: 'sessions.create',
-		proxy: useProxy(url) ? {
-			url: `${config.proxy.host}:${config.proxy.port}`,
-		} : null,
-	}, {
-		encodeJSON: true,
-	});
-
-	if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
-		throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
-	}
-
-	bypassSessions.set(hostname, sessionRes.body.session);
-
-	return sessionRes.body.session;
-}
-
-async function destroyBypassSession(sessionId) {
-	const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
-		cmd: 'sessions.destroy',
-		session: sessionId,
-	}, {
-		encodeJSON: true,
-	}));
-
-	if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
-		bypassSessions.delete(sessionId);
-
-		logger.verbose(`Destroyed bypass session ${sessionId}`);
-
-		return true;
-	}
-
-	logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
-
-	return false;
-}
-
-async function destroyBypassSessions() {
-	if (!config.bypass.cloudflare.enabled) {
-		return;
-	}
-
-	const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
-		cmd: 'sessions.list',
-	}, {
-		encodeJSON: true,
-	}));
-
-	if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
-		logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
-	}
-
-	await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
-}
-
-async function destroyBrowserSessions() {
-	await Promise.all(Array.from(browsers.values(), async (browser) => browser?.close()));
-}
-
-async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
-	const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
-
-	// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
-	const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
-		cmd: `request.${method}`,
-		url,
-		session: sessionId,
-		maxTimeout: options.timeout,
-		proxy: useProxy(url) ? {
-			url: `${config.proxy.host}:${config.proxy.port}`,
-		} : null,
-	}, {
-		encodeJSON: true,
-	}));
-
-	if (!res.statusCode === 200 || res.body?.status !== 'ok') {
-		if (/session closed/i.test(res.body?.message) && attempts < 3) {
-			await destroyBypassSession(sessionId);
-
-			return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
-		}
-
-		throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
-	}
-
-	const resBody = extractJson(res.body.solution.response, res.body.solution.headers);
-
-	return {
-		body: resBody,
-		statusCode: res.body.solution.status,
-		headers: res.body.solution.headers,
-	};
-}
-
-const defaultAgent = new undici.Agent({
-	allowH2: true,
-	connect: {
-		rejectUnauthorized: false,
-	},
-});
-
-const proxyAgent = new undici.ProxyAgent({
-	uri: `http://${config.proxy.host}:${config.proxy.port}`,
-});
-
-async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
-	const withProxy = useProxy(url);
-	const withBrowserBypass = useBrowserBypass(url, requestOptions);
-	const withCloudflareBypass = useCloudflareBypass(url, requestOptions);
-
-	logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
-
-	if (withBrowserBypass) {
-		if (method !== 'get') throw new Error('Browser bypass only supports GET');
-		return bypassBrowserRequest(url, requestOptions);
-	}
-
-	if (withCloudflareBypass) {
-		return bypassCloudflareRequest(url, method, body, withCloudflareBypass, requestOptions);
-	}
-
-	const headers = {
-		...requestOptions.headers,
-	};
-
-	const res = await undici.request(url, {
-		method: method.toUpperCase(),
-		headers,
-		body: body ?? null,
-		dispatcher: withProxy
-			? proxyAgent
-			: defaultAgent,
-		maxRedirections: 0, // handle manually
-	});
-
-	if (res.headers.location && redirects < 3) {
-		// Drain the body to free the socket before redirecting
-		await res.body.dump();
-		const nextUrl = new URL(res.headers.location, url).href;
-		return request(method, nextUrl, body, requestOptions, limiter, redirects + 1);
-	}
-
-	return res;
-}
-
-async function finalizeResult(res, url, options) {
-	if (options.destination) {
-		await pipeline(
-			res.body,
-			...(options.transforms || []),
-			options.destination,
-		);
-
-		return {
-			statusCode: res.statusCode,
-			headers: res.headers,
-			status: res.statusCode,
-			ok: res.statusCode >= 200 && res.statusCode <= 299,
-		};
-	}
-
-	const buffer = await res.body.arrayBuffer();
-	const html = Buffer.from(buffer).toString();
-	const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
-	const pathname = new URL(url).pathname.replace(/\//g, '_');
-
-	if (window && /fetchScene|fetchMovie/.test(new Error().stack)) {
-		windows.set(pathname, window);
-	}
-
-	if (argv.saveHtml) {
-		await fs.writeFile(`./html/${pathname}.html`, html);
-	}
-
-	return {
-		statusCode: res.statusCode,
-		status: res.statusCode,
-		headers: res.headers,
-		body: html,
-		html,
-		document: window?.document || null,
-		window,
-		ok: res.statusCode >= 200 && res.statusCode <= 299,
-	};
-}
-
-function getTimeout(options, url) {
-	return new Promise((_resolve, reject, onCancel) => {
-		const timeout = setTimeout(() => {
-			logger.debug(`Canceled timed out request to ${url}`);
-			reject(new Error(`URL ${url} timed out`));
-		}, (options?.timeout || defaultOptions.timeout) + 10000);
-
-		onCancel(() => {
-			clearTimeout(timeout);
-		});
-	});
-}
-
-async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
-	if (typeof url !== 'string') {
-		console.trace(`Bad URL: ${JSON.stringify(url)}`);
-	}
-
-	const options = {
-		...defaultOptions,
-		...requestOptions,
-		headers: {
-			...Object.fromEntries(requestOptions.includeDefaultHeaders === false ? {} : Object.entries(defaultOptions.headers).map(([key, value]) => [key.toLowerCase(), value])),
-			...Object.fromEntries(Object.entries(requestOptions.headers || {}).map(([key, value]) => [key.toLowerCase(), value])),
-		},
-		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
-		stream: !!requestOptions.destination,
-	};
-
-	const limiter = getLimiter(options, url);
-	const timeout = getTimeout(options, url);
-
-	const result = await limiter.schedule(async () => Promise.race([request(method, url, body, options, limiter), timeout]));
-
-	timeout.cancel();
-
-	const curatedResult = await finalizeResult(result, url, options);
-
-	logger.silly(`Response ${curatedResult.status} for ${method.toUpperCase()} ${url}`);
-
-	return curatedResult;
-}
-
-async function get(url, options) {
-	return scheduleRequest('get', url, null, options);
-}
-
-async function post(url, body, options) {
-	return scheduleRequest('post', url, body, options);
-}
-
-async function put(url, body, options) {
-	return scheduleRequest('put', url, body, options);
-}
-
-async function patch(url, body, options) {
-	return scheduleRequest('patch', url, body, options);
-}
-
-async function del(url, options) {
-	return scheduleRequest('delete', url, null, options);
-}
-
-async function head(url, options) {
-	return scheduleRequest('head', url, null, options);
-}
-
-function getSession(options) {
-	return bhttp.session({ ...defaultOptions, ...options });
-}
-
-function getCookieJar(store, options) {
-	return new toughCookie.CookieJar(store, {
-		looseMode: true,
-		...options,
-	});
-}
-
-module.exports = {
-	toughCookie,
-	get,
-	head,
-	post,
-	delete: del,
-	put,
-	patch,
-	session: getSession,
-	cookieJar: getCookieJar,
-	browsers,
-	getBrowserSession,
-	getBypassSession,
-	getSession,
-	getCookieJar,
-	destroyBypassSessions,
-	destroyBrowserSessions,
-	proxyAgent,
-};

src/utils/img.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const bhttp = require('bhttp');
-const sharp = require('sharp');
-const { argv } = require('yargs');
-
-const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
-
-async function scan() {
-	console.log(url);
-
-	const res = await bhttp.get(url);
-	const stats = await sharp(res.body).stats();
-
-	console.log(stats);
-}
-
-scan();

src/utils/media.js

@@ -1,110 +0,0 @@
-'use strict';
-
-const config = require('config');
-const fs = require('fs');
-const fsPromises = require('fs').promises;
-const Promise = require('bluebird');
-const blake2 = require('blake2');
-const sharp = require('sharp');
-const { nanoid } = require('nanoid');
-const { PassThrough } = require('stream');
-
-const http = require('./http');
-
-function getMemoryUsage() {
-	return process.memoryUsage().rss / (10 ** 6);
-}
-
-let peakMemoryUsage = getMemoryUsage();
-
-async function fetchSource(link) {
-	const id = nanoid();
-
-	const hasher = new blake2.Hash('blake2b');
-	hasher.setEncoding('hex');
-
-	const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
-	const tempFileStream = fs.createWriteStream(tempFilePath);
-	const hashStream = new PassThrough();
-
-	hashStream.on('data', (chunk) => hasher.write(chunk));
-
-	try {
-		const res = await http.get(link, null, {
-			stream: true,
-			transforms: [hashStream],
-			destination: tempFileStream,
-			timeout: 5000,
-		});
-
-		if (!res.ok) {
-			throw new Error(res.status);
-		}
-
-		hasher.end();
-		const hash = hasher.read();
-
-		const memoryUsage = getMemoryUsage();
-		peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
-
-		console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
-
-		return {
-			id,
-			path: tempFilePath,
-			hash,
-		};
-	} catch (error) {
-		await fsPromises.unlink(tempFilePath);
-
-		throw error;
-	}
-}
-
-async function init() {
-	const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
-	const links = linksFile.split('\n').filter(Boolean);
-
-	await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
-
-	console.time('thumbs');
-
-	const files = await Promise.map(links, async (link) => {
-		try {
-			return await fetchSource(link);
-		} catch (error) {
-			console.log(`Failed to fetch ${link}: ${error.message}`);
-			return null;
-		}
-	});
-
-	await Promise.map(files.filter(Boolean), async (file) => {
-		const image = sharp(file.path).jpeg();
-
-		const [{ width, height }, { size }] = await Promise.all([
-			image.metadata(),
-			fsPromises.stat(file.path),
-		]);
-
-		await Promise.all([
-			image
-				.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
-			image
-				.resize({
-					height: config.media.thumbnailSize,
-					withoutEnlargement: true,
-				})
-				.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
-		]);
-
-		const memoryUsage = getMemoryUsage();
-		peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
-
-		console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
-	}, { concurrency: 10 });
-
-	console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
-	console.timeEnd('thumbs');
-}
-
-init();

src/utils/mofos.js

@@ -1,21 +0,0 @@
-'use strict';
-
-const Promise = require('bluebird');
-const bhttp = require('bhttp');
-
-const knex = require('../knex');
-
-async function run() {
-	const network = await knex('networks').where('slug', 'mofos').first();
-	const sites = await knex('sites').where('network_id', network.id);
-
-	await Promise.map(sites, async (site) => {
-		const res = await bhttp.get(site.url);
-
-		console.log(site.url, res.statusCode);
-	}, {
-		concurrency: 5,
-	});
-}
-
-run();

src/utils/q.js

@@ -1,5 +0,0 @@
-'use strict';
-
-const qu = require('./qu');
-
-module.exports = qu;

src/utils/qu.js

@@ -1,682 +0,0 @@
-'use strict';
-
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-const http = require('./http');
-const virtualConsole = require('./virtual-console')(__filename);
-
-const { window: globalWindow } = new JSDOM('', { virtualConsole });
-
-function trim(str) {
-	if (typeof str !== 'string') {
-		return str;
-	}
-
-	return str.trim().replace(/\s+/g, ' ');
-}
-
-function extractDate(dateString, format, match) {
-	if (!dateString) {
-		return null;
-	}
-
-	if (match) {
-		const dateStamp = trim(dateString).match(match);
-
-		if (dateStamp) {
-			const dateValue = moment.utc(dateStamp[0], format);
-
-			return dateValue.isValid() ? dateValue.toDate() : null;
-		}
-		return null;
-	}
-
-	const dateValue = moment.utc(trim(dateString), format);
-
-	return dateValue.isValid() ? dateValue.toDate() : null;
-}
-
-function formatDate(dateValue, format, inputFormat) {
-	if (inputFormat) {
-		return moment(dateValue, inputFormat).format(format);
-	}
-
-	return moment(dateValue).format(format);
-}
-
-function durationToSeconds(durationString, match) {
-	const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
-
-	if (durationMatch) {
-		const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3);
-
-		return moment.duration(segments.join(':')).asSeconds();
-	}
-
-	return null;
-}
-
-function prefixUrl(urlValue, origin, protocol = 'https') {
-	if (!urlValue) {
-		return null;
-	}
-
-	if (/^http/.test(urlValue)) {
-		return urlValue;
-	}
-
-	if (protocol && /^\/\//.test(urlValue)) {
-		return `${protocol}:${urlValue}`;
-	}
-
-	if (origin && /^\//.test(urlValue)) {
-		return `${origin}${urlValue}`;
-	}
-
-	if (origin && /^\.\//.test(urlValue)) {
-		return `${origin}${urlValue.slice(1)}`;
-	}
-
-	if (origin) {
-		return `${origin}/${urlValue}`;
-	}
-
-	return urlValue;
-}
-
-function iterateXPathResult(iterator, results = []) {
-	const element = iterator.iterateNext();
-
-	if (element) {
-		return iterateXPathResult(iterator, [...results, element]);
-	}
-
-	return results;
-}
-
-function getElements(context, selector, first = false) {
-	if (!selector) {
-		return context;
-	}
-
-	if (/^\/\//.test(selector)) {
-		// XPath selector
-		const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
-
-		if (first) {
-			return iterator.iterateNext();
-		}
-
-		return iterateXPathResult(iterator);
-	}
-
-	if (first) {
-		return context.querySelector(selector);
-	}
-
-	return Array.from(context.querySelectorAll(selector));
-}
-
-function q(context, selectors, attrArg, applyTrim = true) {
-	if (!selectors && context.nodeName === '#document') {
-		return null;
-	}
-
-	const attr = attrArg === true ? 'textContent' : attrArg;
-	const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
-
-	if (!element) {
-		return null;
-	}
-
-	if (attr) {
-		const value = element[attr] || element.getAttribute(attr);
-
-		return applyTrim && typeof value === 'string' ? trim(value) : value;
-	}
-
-	return element;
-}
-
-function all(context, selectors, attrArg, applyTrim = true) {
-	const attr = attrArg === true ? 'textContent' : attrArg;
-	const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null);
-
-	if (!Array.isArray(elements)) {
-		return [];
-	}
-
-	if (attr) {
-		return elements.map((el) => q(el, null, attr, applyTrim));
-	}
-
-	return elements;
-}
-
-function exists(context, selector) {
-	return !!q(context, selector);
-}
-
-function count(context, selector) {
-	return all(context, selector)?.length || 0;
-}
-
-function content(context, selector, applyTrim = true) {
-	return q(context, selector, 'textContent', applyTrim);
-}
-
-function contents(context, selector, applyTrim) {
-	return all(context, selector, 'textContent', applyTrim);
-}
-
-function html(context, selector) {
-	const el = q(context, selector, null, true);
-
-	return el && el.innerHTML;
-}
-
-function htmls(context, selector) {
-	const els = all(context, selector, null, true);
-
-	return els.map((el) => el.innerHTML);
-}
-
-function execute(context, selector = 'script') {
-	const scripts = htmls(context, selector);
-	const originalGlobal = Object.fromEntries(Object.entries(global));
-
-	const errors = scripts?.reduce((accErrors, script) => {
-		try {
-			Function(script)(); /* eslint-disable-line no-new-func */
-
-			return accErrors;
-		} catch (error) {
-			// the script failed
-			return [...accErrors, error];
-		}
-	}, []);
-
-	const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => {
-		if (originalGlobal[key] !== value) {
-			delete global[key];
-			return true;
-		}
-
-		return false;
-	}));
-
-	return {
-		...data,
-		errors,
-	};
-}
-
-function json(context, selector) {
-	const el = q(context, selector, null, true);
-
-	try {
-		return JSON.parse(el?.innerHTML);
-	} catch (error) {
-		return null;
-	}
-}
-
-function jsons(context, selector) {
-	const els = all(context, selector, null, true);
-
-	return els.map((el) => {
-		try {
-			return JSON.parse(el?.innerHTML);
-		} catch (error) {
-			return null;
-		}
-	});
-}
-
-function texts(context, selector, applyTrim = true, filter = true) {
-	const el = q(context, selector, null, applyTrim);
-	if (!el) return null;
-
-	const nodes = Array.from(el.childNodes)
-		.filter((node) => node.nodeName === '#text')
-		.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
-
-	return filter ? nodes.filter(Boolean) : nodes;
-}
-
-function text(context, selector, applyTrim = true) {
-	const nodes = texts(context, selector, applyTrim, true);
-	if (!nodes) return null;
-
-	const textValue = nodes.join(' ');
-
-	return applyTrim ? trim(textValue) : textValue;
-}
-
-function removeStyleFunctionSpaces(el) {
-	// jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png )
-	el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`));
-}
-
-function style(context, selector, styleAttr) {
-	const el = q(context, selector);
-
-	if (el?.hasAttribute('style')) {
-		removeStyleFunctionSpaces(el);
-
-		return styleAttr ? el.style[styleAttr] : el.style;
-	}
-
-	return null;
-}
-
-function styles(context, selector, styleAttr) {
-	const elStyles = Array.from(context.querySelectorAll(selector), (el) => {
-		removeStyleFunctionSpaces(el);
-
-		return styleAttr ? el.style[styleAttr] : el.style;
-	});
-
-	return elStyles;
-}
-
-function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
-	const value = q(context, selector, attr);
-
-	if (value && match) {
-		return Number(value.match(match)?.[matchIndex]);
-	}
-
-	if (value) {
-		return Number(value);
-	}
-
-	return null;
-}
-
-function meta(context, selector, attrArg = 'content', applyTrim = true) {
-	if (/meta\[.*\]/.test(selector)) {
-		return q(context, selector, attrArg, applyTrim);
-	}
-
-	return q(context, `meta[${selector}]`, attrArg, applyTrim);
-}
-
-function date(context, selector, format, match, attr = 'textContent') {
-	const dateString = q(context, selector, attr, true);
-
-	if (!dateString) return null;
-
-	return extractDate(dateString, format, match);
-}
-
-function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') {
-	const timeString = q(context, selector, attr, 'textContent');
-
-	if (!timeString) {
-		return null;
-	}
-
-	const timeMatch = timeString.match(match);
-
-	if (timeMatch) {
-		const [n, period] = timeMatch.slice(1);
-		const thenDate = moment.utc().subtract(Number(n), period);
-
-		return {
-			date: thenDate.toDate(),
-			precision: period.replace(/s$/, ''),
-		};
-	}
-
-	return null;
-}
-
-function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
-	const imageEl = (attr && q(context, selector, attr))
-		|| q(context, selector, 'data-src')
-		|| q(context, selector, 'src');
-
-	return prefixUrl(imageEl, origin, protocol);
-}
-
-function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
-	const attribute = attr
-		|| (q(context, selector, 'data-src') && 'data-src')
-		|| (q(context, selector, 'src') && 'src');
-
-	const imageEls = all(context, selector, attribute);
-
-	return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
-}
-
-function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
-	const urlEl = q(context, selector, attr);
-	const prefixedUrl = prefixUrl(urlEl, origin, protocol);
-
-	if (prefixedUrl && object) {
-		return new URL(prefixedUrl);
-	}
-
-	return prefixedUrl;
-}
-
-function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
-	const urlEls = all(context, selector, attr);
-
-	return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
-}
-
-function sourceSet(context, selector, attr = 'srcset', options = {}) {
-	const srcset = q(context, selector, attr);
-
-	if (!srcset) {
-		return null;
-	}
-
-	const sources = srcset
-		.split(/\s*,\s*/)
-		.map((source) => {
-			const [link, descriptor] = source.split(' ');
-
-			if (link) {
-				return {
-					descriptor: descriptor || 'fallback',
-					url: prefixUrl(link, options.origin, options.protocol),
-				};
-			}
-
-			return null;
-		})
-		.filter(Boolean)
-		.sort((sourceA, sourceB) => {
-			if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) {
-				return -1;
-			}
-
-			if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) {
-				return 1;
-			}
-
-			return 0;
-		});
-
-	if (options.includeDescriptor) {
-		return sources;
-	}
-
-	return sources.map((source) => source.url);
-}
-
-function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
-	const posterEl = q(context, selector, attr);
-
-	return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
-}
-
-function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
-	const trailerEl = q(context, selector, attr);
-
-	return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
-}
-
-function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
-	const trailerEls = all(context, selector, attr);
-
-	return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
-}
-
-function duration(context, selector, match, attr = 'textContent') {
-	const durationString = q(context, selector, attr);
-
-	if (!durationString) {
-		return null;
-	}
-
-	const durationMatch = durationToSeconds(durationString, match);
-
-	if (durationMatch) {
-		return durationMatch;
-	}
-
-	const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i);
-
-	if (timestampMatch) {
-		const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0;
-		const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0;
-		const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0;
-
-		return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds);
-	}
-
-	return null;
-}
-
-const legacyFuncs = {
-	q,
-	qa: all,
-	qall: all,
-	qd: date,
-	qdate: date,
-	qh: html,
-	qhtml: html,
-	qi: image,
-	qimage: image,
-	qimages: images,
-	qis: images,
-	ql: duration,
-	qlength: duration,
-	qm: meta,
-	qmeta: meta,
-	qp: poster,
-	qposter: poster,
-	qs: all,
-	qt: video,
-	qtext: text,
-	qtexts: texts,
-	qtrailer: video,
-	qtrailers: videos,
-	qts: videos,
-	qtx: text,
-	qtxs: texts,
-	qtxt: text,
-	qtxts: texts,
-	// qu: url,
-	qurl: url,
-	qurls: urls,
-	qus: urls,
-};
-
-const quFuncs = {
-	all,
-	cnt: content,
-	cnts: contents,
-	content,
-	contents,
-	count,
-	date,
-	dateAgo,
-	dur: duration,
-	duration,
-	el: q,
-	element: q,
-	execute,
-	exec: execute,
-	exists,
-	html,
-	htmls,
-	image,
-	images,
-	img: image,
-	imgs: images,
-	json,
-	jsons,
-	length: duration,
-	meta,
-	num: number,
-	number,
-	poster,
-	q,
-	sourceSet,
-	sources: sourceSet,
-	srcs: sourceSet,
-	srcset: sourceSet,
-	style,
-	styles,
-	text,
-	texts,
-	trailer: video,
-	url,
-	urls,
-	video,
-	videos,
-};
-
-function init(context, selector, window) {
-	if (!context) {
-		return null;
-	}
-
-	const element = selector ? context.querySelector(selector) : context;
-
-	if (!element) {
-		return null;
-	}
-
-	const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
-		.reduce((acc, [key, func]) => ({
-			...acc,
-			[key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context
-				? func(...args)
-				: func(element, ...args)),
-		}), {});
-
-	const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
-		.reduce((acc, [key, func]) => ({
-			...acc,
-			[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
-				? func(element, ...args)
-				: func(...args)),
-		}), {});
-
-	return {
-		element,
-		el: element,
-		html: element.outerHTML || element.body.outerHTML,
-		text: trim(element.textContent),
-		...(window && {
-			window,
-			document: window.document,
-		}),
-		...legacyContextFuncs,
-		qu: quContextFuncs,
-		query: quContextFuncs,
-	};
-}
-
-function initAll(context, selector, window) {
-	if (Array.isArray(context)) {
-		return context.map((element) => init(element, null, window));
-	}
-
-	return Array.from(context.querySelectorAll(selector))
-		.map((element) => init(element, null, window));
-}
-
-function extract(htmlValue, selector, options) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
-
-	return init(window.document, selector, window);
-}
-
-function extractAll(htmlValue, selector, options) {
-	const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
-
-	return initAll(window.document, selector, window);
-}
-
-async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
-	const res = await (method === 'post'
-		? http.post(urlValue, body, { ...options, headers })
-		: http[method](urlValue, {
-			...options,
-			headers,
-			parse: true,
-		}));
-
-	if (res.ok) {
-		const item = queryAll
-			? initAll(res.document, selector, res.window)
-			: init(res.document, selector, res.window);
-
-		return {
-			item,
-			items: all ? item : [item],
-			res,
-			ok: true,
-			status: res.statusCode,
-			...res,
-		};
-	}
-
-	return {
-		item: null,
-		items: [],
-		res,
-		ok: false,
-		status: res.statusCode,
-		...res,
-	};
-}
-
-async function get(urlValue, selector, headers, options) {
-	return request('get', urlValue, null, selector, headers, options, false);
-}
-
-async function post(urlValue, body, selector, headers, options) {
-	return request('post', urlValue, body, selector, headers, options, false);
-}
-
-async function getAll(urlValue, selector, headers, options) {
-	return request('get', urlValue, null, selector, headers, options, true);
-}
-
-async function postAll(urlValue, body, selector, headers, options) {
-	return request('post', urlValue, body, selector, headers, options, true);
-}
-
-function session(headers, options) {
-	return http.session(headers, options);
-}
-
-module.exports = {
-	extractDate,
-	extract,
-	extractAll,
-	durationToSeconds,
-	init,
-	initAll,
-	formatDate,
-	get,
-	getAll,
-	http,
-	fetch: get,
-	fetchAll: getAll,
-	context: init,
-	contextAll: initAll,
-	ed: extractDate,
-	ex: extract,
-	exa: extractAll,
-	fd: formatDate,
-	parseDate: extractDate,
-	ctx: init,
-	ctxa: initAll,
-	geta: getAll,
-	qu: quFuncs,
-	query: quFuncs,
-	post,
-	postAll,
-	prefixUrl,
-	session,
-	...legacyFuncs,
-};

src/utils/scorelogos.js

@@ -1,37 +0,0 @@
-'use strict';
-
-const Promise = require('bluebird');
-const bhttp = require('bhttp');
-const fs = require('fs-extra');
-const knex = require('../knex');
-
-async function init() {
-	const sites = await knex('sites')
-		.select('networks.name', 'sites.slug')
-		.join('networks', 'networks.id', 'sites.network_id')
-		.where('networks.slug', 'score');
-
-	await Promise.map(sites, async (site) => {
-		const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
-
-		console.log(url);
-
-		const res = await bhttp.get(url, {
-			responseTimeout: 5000,
-		});
-
-		if (res.statusCode === 200) {
-			console.log(`Saving logo for ${site.slug}`);
-
-			await fs.writeFile(`./score/${site.slug}.png`, res.body);
-		}
-
-		console.log(`No logo found for ${site.slug}`);
-	}, {
-		concurrency: 10,
-	});
-
-	knex.destroy();
-}
-
-init();

src/utils/stream.js

@@ -1,55 +0,0 @@
-'use strict';
-
-const config = require('config');
-const { PassThrough } = require('stream');
-const fs = require('fs');
-const path = require('path');
-const bhttp = require('bhttp');
-const blake2 = require('blake2');
-const sharp = require('sharp');
-
-const url = 'https://thumbs.julesjordan.com/trial/content//upload/dl03/julesjordan/oil_overload_16_scene2//photos/alina_lopez_jules_jordan_com_77.jpg';
-
-async function init() {
-	const hash = new blake2.Hash('blake2b');
-	hash.setEncoding('hex');
-
-	const res = await bhttp.get(url, {
-		stream: true,
-	});
-
-	const metaStream = sharp();
-	const hashStream = new PassThrough();
-	const target = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina.jpg'));
-	const thumbTarget = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina_thumb.jpg'));
-
-	hashStream.on('data', (chunk) => {
-		hash.write(chunk);
-	});
-
-	metaStream.clone()
-		.resize(320)
-		.pipe(thumbTarget);
-
-	const stream = res
-		.pipe(metaStream)
-		.pipe(hashStream)
-		.pipe(target);
-
-	stream.on('finish', () => {
-		hash.end();
-		const digest = hash.read();
-
-		console.log('stream', digest);
-	});
-
-	metaStream.on('info', (info) => {
-		console.log('info', info);
-	});
-
-	const stats = await metaStream.stats();
-
-	console.log('stats', stats);
-}
-
-init();

src/utils/tcc.js

@@ -1,827 +0,0 @@
-const Promise = require('bluebird');
-const bhttp = require('bhttp');
-const fs = require('fs');
-
-const channels = 	[{
-	name: 'Analyzed Girls',
-	slug: 'analyzedgirls',
-	url: 'https://analyzedgirls.com',
-	alias: ['ag',
-	],
-	tags: ['anal',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 178,
-	},
-},
-{
-	name: 'Archived Updates',
-	slug: 'archivedupdates',
-	url: 'https://archivedupdates.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 482,
-	},
-},
-{
-	name: 'Ass Teen Mouth',
-	slug: 'assteenmouth',
-	url: 'https://assteenmouth.com',
-	alias: ['atm',
-	],
-	tags: ['anal',
-		'atm',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 180,
-	},
-},
-{
-	name: 'Bang Teen Pussy',
-	slug: 'bangteenpussy',
-	url: 'https://bangteenpussy.com',
-	alias: ['btp',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 182,
-	},
-},
-{
-	name: 'Brutal Invasion',
-	slug: 'brutalinvasion',
-	url: 'https://brutalinvasion.com',
-	alias: ['bin',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 184,
-	},
-},
-{
-	name: 'Clean My Ass',
-	slug: 'cleanmyass',
-	url: 'https://cleanmyass.com',
-	tags: ['ass-eating'],
-	alias: ['cma',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 362,
-	},
-},
-{
-	name: 'College Party Time',
-	slug: 'collegepartytime',
-	url: 'https://collegepartytime.com',
-	alias: ['cpt',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 278,
-	},
-},
-{
-	name: 'Cumaholic Teens',
-	slug: 'cumaholicteens',
-	url: 'https://cumaholicteens.com',
-	alias: ['ct',
-		'cumoholic teens',
-		'cumoholicteens',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 186,
-	},
-},
-{
-	name: 'Cum Filled Throat',
-	slug: 'cumfilledthroat',
-	url: 'https://cumfilledthroat.com',
-	alias: ['cft',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 280,
-	},
-},
-{
-	name: 'Cum Sumption Cocktail',
-	slug: 'cumsumptioncocktail',
-	url: 'https://cumsumptioncocktail.com',
-	alias: ['csc',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 282,
-	},
-},
-{
-	name: 'Defiled18',
-	slug: 'defiled18',
-	url: 'https://defiled18.com',
-	alias: ['d18',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 188,
-	},
-},
-{
-	name: 'Dirty Baby Sitter',
-	slug: 'dirtybabysitter',
-	url: 'https://dirtybabysitter.com',
-	alias: ['dbs',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 284,
-	},
-},
-{
-	name: 'Double Teamed Teens',
-	slug: 'doubleteamedteens',
-	url: 'https://doubleteamedteens.com',
-	alias: ['dtt',
-	],
-	tags: ['teen',
-		'dp',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 190,
-	},
-},
-{
-	name: 'Dreamteens HD',
-	slug: 'dreamteenshd',
-	url: 'https://dreamteenshd.com',
-	alias: ['dhd',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 192,
-	},
-},
-{
-	name: 'Drill Her Butt',
-	slug: 'drillherbutt',
-	url: 'https://drillherbutt.com',
-	tags: ['anal'],
-	alias: ['dhb',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 288,
-	},
-},
-{
-	name: 'Dual Throat',
-	slug: 'dualthroat',
-	url: 'https://dualthroat.com',
-	alias: ['dt',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 290,
-	},
-},
-{
-	name: 'Gang Land Victims',
-	slug: 'ganglandvictims',
-	url: 'https://ganglandvictims.com',
-	alias: ['glv',
-	],
-	tags: ['gangbang',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 292,
-	},
-},
-{
-	name: 'Girls Got Cream',
-	slug: 'girlsgotcream',
-	url: 'https://girlsgotcream.com',
-	alias: ['ggc',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 194,
-	},
-},
-{
-	name: 'Hardcore Youth',
-	slug: 'hardcoreyouth',
-	url: 'https://hardcoreyouth.com',
-	alias: ['hcy',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 196,
-	},
-},
-{
-	name: 'Little Hellcat',
-	slug: 'littlehellcat',
-	url: 'https://littlehellcat.com',
-	alias: ['lhc',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 198,
-	},
-},
-{
-	name: 'Little Teen Suckers',
-	slug: 'littleteensuckers',
-	url: 'https://littleteensuckers.com',
-	alias: ['lts',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 294,
-	},
-},
-{
-	name: 'MAIKO',
-	slug: 'maiko',
-	url: 'https://maiko.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 566,
-	},
-},
-{
-	name: 'Make Teen Gape',
-	slug: 'maketeengape',
-	url: 'https://maketeengape.com',
-	alias: ['mtg',
-	],
-	tags: ['teen',
-		'gaping',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 200,
-	},
-},
-{
-	name: 'Make Teen Moan',
-	slug: 'maketeenmoan',
-	url: 'https://maketeenmoan.com',
-	alias: ['mtm',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 296,
-	},
-},
-{
-	name: 'Mega Penetrations',
-	slug: 'megapenetrations',
-	url: 'https://megapenetrations.com',
-	alias: ['mp',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 298,
-	},
-},
-{
-	name: 'Messy Gang Bangs',
-	slug: 'messygangbangs',
-	url: 'https://messygangbangs.com',
-	alias: ['mgb',
-	],
-	tags: ['gangbang',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 300,
-	},
-},
-{
-	name: 'My Black Coeds',
-	slug: 'myblackcoeds',
-	url: 'https://myblackcoeds.com',
-	alias: ['mbc',
-	],
-	tags: ['ebony',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 302,
-	},
-},
-{
-	name: 'My Latina Teen',
-	slug: 'mylatinateen',
-	url: 'https://mylatinateen.com',
-	alias: ['mlt',
-	],
-	tags: ['teen',
-		'latina',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 304,
-	},
-},
-{
-	name: 'Nasty Ass Lickers',
-	slug: 'nastyasslickers',
-	url: 'https://nastyasslickers.com',
-	tags: ['ass-eating'],
-	alias: ['nal',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 306,
-	},
-},
-{
-	name: 'Naughty Little Nymphs',
-	slug: 'naughtylittlenymphs',
-	url: 'https://naughtylittlenymphs.com',
-	alias: ['nln',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 308,
-	},
-},
-{
-	name: 'NC UK Sinners',
-	slug: 'ncuksinners',
-	url: 'https://ncuksinners.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 568,
-	},
-},
-{
-	name: 'NC Verso Cinema',
-	slug: 'ncversocinema',
-	url: 'https://ncversocinema.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 570,
-	},
-},
-{
-	name: 'Network Mixed',
-	slug: 'networkmixed',
-	url: 'https://networkmixed.com',
-	alias: ['anc',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 360,
-	},
-},
-{
-	name: 'Never Done That Before',
-	slug: 'neverdonethatbefore',
-	url: 'https://neverdonethatbefore.com',
-	alias: ['ndt',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 310,
-	},
-},
-{
-	name: 'Nylon Sweeties',
-	slug: 'nylonsweeties',
-	url: 'https://nylonsweeties.com',
-	alias: ['nsw',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 202,
-	},
-},
-{
-	name: 'Pink Eye Sluts',
-	slug: 'pinkeyesluts',
-	url: 'https://pinkeyesluts.com',
-	alias: ['pes',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 312,
-	},
-},
-{
-	name: 'Plug2 Holes',
-	slug: 'plug2holes',
-	url: 'https://plug2holes.com',
-	alias: ['p2h',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 314,
-	},
-},
-{
-	name: 'Pussy Babes',
-	slug: 'pussybabes',
-	url: 'https://pussybabes.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 556,
-	},
-},
-{
-	name: 'Road Gang Bangs',
-	slug: 'roadgangbangs',
-	url: 'https://roadgangbangs.com',
-	alias: ['rgb',
-	],
-	tags: ['gangbang',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 316,
-	},
-},
-{
-	name: 'Russian Teen Updates',
-	slug: 'russianteenupdates',
-	url: 'https://russianteenupdates.com',
-	tags: ['teen',
-		'russian',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 418,
-	},
-},
-{
-	name: 'School Bus Chicks',
-	slug: 'schoolbuschicks',
-	url: 'https://schoolbuschicks.com',
-	alias: ['sbc',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 318,
-	},
-},
-{
-	name: 'Seductive18',
-	slug: 'seductive18',
-	url: 'https://seductive18.com',
-	alias: ['s18',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 204,
-	},
-},
-{
-	name: 'Show Me Gape',
-	slug: 'showmegape',
-	url: 'https://showmegape.com',
-	tags: ['gaping'],
-	alias: ['smg',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 320,
-	},
-},
-{
-	name: 'Shy Teachers Pet',
-	slug: 'shyteacherspet',
-	url: 'https://shyteacherspet.com',
-	alias: ['stp',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 322,
-	},
-},
-{
-	name: 'Small Tits Hunter',
-	slug: 'smalltitshunter',
-	url: 'https://smalltitshunter.com',
-	alias: ['sth',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 324,
-	},
-},
-{
-	name: 'Spermantino',
-	slug: 'spermantino',
-	url: 'https://spermantino.com',
-	alias: ['spe',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 366,
-	},
-},
-{
-	name: 'TCC Network Archives',
-	slug: 'tccnetworkarchives',
-	url: 'https://tccnetworkarchives.com',
-	alias: ['all',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 176,
-	},
-},
-{
-	name: 'Teach My Ass',
-	slug: 'teachmyass',
-	url: 'https://teachmyass.com',
-	tags: ['anal'],
-	alias: ['tma',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 368,
-	},
-},
-{
-	name: 'Teenagers Going Wild',
-	slug: 'teenagersgoingwild',
-	url: 'https://teenagersgoingwild.com',
-	alias: ['tgw',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 326,
-	},
-},
-{
-	name: 'Teen Anal Casting',
-	slug: 'teenanalcasting',
-	url: 'https://teenanalcasting.com',
-	alias: ['tac',
-	],
-	tags: ['anal',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 206,
-	},
-},
-{
-	name: 'Teen Drillers',
-	slug: 'teendrillers',
-	url: 'https://teendrillers.com',
-	alias: ['td',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 208,
-	},
-},
-{
-	name: 'Teen Gina',
-	slug: 'teengina',
-	url: 'https://teengina.com',
-	alias: ['tg',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 210,
-	},
-},
-{
-	name: 'Teens Love Blacks',
-	slug: 'teensloveblacks',
-	url: 'https://teensloveblacks.com',
-	alias: ['tlb',
-	],
-	tags: ['interracial',
-		'bbc',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 328,
-	},
-},
-{
-	name: 'Teens Natural Way',
-	slug: 'teensnaturalway',
-	url: 'https://teensnaturalway.com',
-	alias: ['tnw',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 212,
-	},
-},
-{
-	name: 'Teens Try Anal',
-	slug: 'teenstryanal',
-	url: 'https://teenstryanal.com',
-	alias: ['tta',
-	],
-	tags: ['anal',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 330,
-	},
-},
-{
-	name: 'Teens Try Blacks',
-	slug: 'teenstryblacks',
-	url: 'https://teenstryblacks.com',
-	alias: ['ttb',
-	],
-	tags: ['interracial',
-		'bbc',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 214,
-	},
-},
-{
-	name: 'Teens Want Orgies',
-	slug: 'teenswantorgies',
-	url: 'https://teenswantorgies.com',
-	alias: ['two',
-	],
-	tags: ['teen',
-		'threesome',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 332,
-	},
-},
-{
-	name: 'Try Teens',
-	slug: 'tryteens',
-	url: 'https://tryteens.com',
-	alias: ['tt',
-	],
-	tags: ['teen',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 216,
-	},
-},
-{
-	name: 'Tug Job Queens',
-	slug: 'tugjobqueens',
-	url: 'https://tugjobqueens.com',
-	tags: ['handjob'],
-	alias: ['tjq',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 334,
-	},
-},
-{
-	name: 'UK Sinners',
-	slug: 'uksinners',
-	url: 'https://uksinners.com',
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 558,
-	},
-},
-{
-	name: 'White Box Black Cocks',
-	slug: 'whiteboxblackcocks',
-	url: 'https://whiteboxblackcocks.com',
-	alias: ['wbc',
-	],
-	tags: ['interracial',
-		'bbc',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 336,
-	},
-},
-{
-	name: 'White Teens Black Cocks',
-	slug: 'whiteteensblackcocks',
-	url: 'https://whiteteensblackcocks.com',
-	alias: ['wbc',
-	],
-	tags: ['interracial',
-		'bbc',
-		'teen',
-	],
-	parent: 'teencoreclub',
-	hasLogo: false,
-	parameters: {
-		siteId: 218,
-	},
-},
-{
-	name: 'Young Throats',
-	slug: 'youngthroats',
-	url: 'https://youngthroats.com',
-	tags: ['blowjob', 'deepthroat', 'facefucking'],
-	alias: ['yt',
-	],
-	parent: 'teencoreclub',
-	parameters: {
-		siteId: 220,
-	},
-}];
-
-async function init() {
-	const results = await Promise.map(channels, async (channel) => {
-		try {
-			const res = await bhttp.head(channel.url, {
-				followRedirects: false,
-				responseTimeout: 30000,
-			});
-
-			console.log(res.statusCode, channel.url);
-
-			if (res.statusCode === 200) {
-				return channel;
-			}
-
-			throw new Error(`${channel.url} no longer available: ${res.statusCode}`);
-		} catch (error) {
-			console.error(channel.url, error.message);
-
-			return {
-				name: channel.name,
-				slug: channel.slug,
-				url: `https://teencoreclub.com/browsevideos/site/${channel.parameters.siteId}/${channel.name.replace(/\s+/g, '')}`,
-				...(channel.tags && { tags: channel.tags }),
-				...(channel.alias && { alias: channel.alias }),
-				parent: channel.parent,
-				...(channel.hasLogo !== undefined && { hasLogo: channel.hasLogo }),
-				parameters: channel.parameters,
-			};
-		}
-	}, { concurrency: 5 });
-
-	await fs.promises.writeFile('./tcc.json', JSON.stringify(results, null, 4));
-
-	console.log('done!');
-}
-
-init();

src/utils/timeout.js

@@ -1,20 +0,0 @@
-'use strict';
-
-const http = require('./http');
-
-const sleep = 5000;
-const timeout = 1000;
-
-async function init() {
-	try {
-		const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
-			timeout,
-		});
-
-		console.log(res.statusCode);
-	} catch (error) {
-		console.log(error);
-	}
-}
-
-init();

src/utils/virtual-console.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const { VirtualConsole } = require('jsdom');
-const path = require('path');
-
-const logger = require('../logger')(__filename);
-
-function getVirtualConsole(filepath) {
-	const virtualConsole = new VirtualConsole();
-	const context = path.basename(filepath).replace(path.extname(filepath), '');
-
-	virtualConsole.on('error', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
-	virtualConsole.on('jsdomError', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
-
-	return virtualConsole;
-}
-
-module.exports = getVirtualConsole;

tests/profiles.js

@@ -266,6 +266,7 @@ const actors = [
 	{ entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
 	{ entity: 'darkkotv', name: 'Aidra Fox', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'measurements', 'naturalBoobs', 'hasTattoos', 'hasPiercings'] },
 	{ entity: 'jaxslayher', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'eyes', 'height', 'weight', 'birthPlace', 'measurements', 'hairColor'] },
+	{ entity: 'cosplayground', name: 'Anna Claire Clouds', url: 'https://cosplayground.com/models/anna-claire-clouds?models=Anna%20Claire%20Clouds', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'height', 'measurements', 'hairColor', 'ethnicity'] },
 ];
 
 const actorScrapers = scrapers.actors;