1.255.9

config/default.js

@@ -181,6 +181,8 @@ module.exports = {
 			'traxxx',
 			// porn doe
 			'forbondage',
+			// prevent huge influx
+			'manyvids',
 		],
 	},
 	profiles: null,
@@ -199,6 +201,12 @@ module.exports = {
 			// source: 'http://nsfw.unknown.name/random',
 		},
 	},
+	webApi: {
+		enabled: true,
+		address: 'http://localhost:5100/api',
+		apiUserId: 1,
+		apiKey: null,
+	},
 	proxy: {
 		enable: false,
 		test: 'https://api.ipify.org?format=json',
@@ -280,6 +288,9 @@ module.exports = {
 			excludeHostnames: [],
 			selectIndex: {},
 		},
+		biometrics: {
+			enabled: true,
+		},
 	},
 	titleSlugLength: 50,
 };

migrations/20190325001339_releases.js

@@ -40,6 +40,7 @@ exports.up = async (knex) => {
 			.index();
 
 		table.text('name');
+		table.string('name_stylized');
 		table.text('slug', 32);
 
 		table.text('type')
@@ -76,7 +77,7 @@ exports.up = async (knex) => {
 	});
 
 	await knex.schema.createTable('media', (table) => {
-		table.text('id', 21)
+		table.string('id', 21)
 			.primary();
 
 		table.text('path');
@@ -126,10 +127,15 @@ exports.up = async (knex) => {
 			.defaultTo(knex.fn.now());
 	});
 
+	await knex.schema.createMaterializedView('media_sfw', (view) => {
+		view.as(knex('media').select('id').where('is_sfw', true));
+	});
+
+	await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
+
 	await knex.raw(`
 		CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media
+			SELECT id FROM media_sfw
-			WHERE is_sfw = true
 			ORDER BY random()
 			LIMIT 1;
 		$$ LANGUAGE sql STABLE;
@@ -143,6 +149,36 @@ exports.up = async (knex) => {
 			.defaultTo(knex.raw('get_random_sfw_media_id()'));
 	});
 
+	await knex.schema.createTable('media_biometrics', (table) => {
+		table.increments('id');
+
+		table.string('media_id', 21)
+			.references('id')
+			.inTable('media')
+			.notNullable()
+			.onDelete('cascade');
+
+		table.integer('width');
+		table.integer('height');
+
+		table.integer('face_index')
+			.notNullable()
+			.defaultTo(0);
+
+		table.json('biometrics');
+		table.specificType('embedding', 'vector(1024)');
+
+		table.datetime('updated_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+
+		table.unique(['media_id', 'face_index']);
+	});
+
 	await knex.schema.createTable('tags_groups', (table) => {
 		table.increments('id');
 
@@ -335,7 +371,7 @@ exports.up = async (knex) => {
 
 		table.decimal('shoe_size');
 		table.integer('leg');
-		table.integer('foot');
+		table.decimal('foot');
 		table.integer('thigh');
 
 		table.string('blood_type');
@@ -357,6 +393,9 @@ exports.up = async (knex) => {
 
 		table.string('agency');
 
+		table.boolean('allow_global_match');
+		table.text('comment');
+
 		table.text('avatar_media_id', 21)
 			.references('id')
 			.inTable('media');
@@ -443,7 +482,7 @@ exports.up = async (knex) => {
 
 		table.decimal('shoe_size');
 		table.integer('leg');
-		table.integer('foot');
+		table.decimal('foot');
 		table.integer('thigh');
 
 		table.string('blood_type');
@@ -655,6 +694,7 @@ exports.up = async (knex) => {
 			.inTable('actors');
 
 		table.integer('profile_id', 12)
+			.notNullable()
 			.references('id')
 			.inTable('actors_profiles')
 			.onDelete('cascade');
@@ -672,8 +712,6 @@ exports.up = async (knex) => {
 		table.unique(['profile_id', 'media_id']);
 	});
 
-	await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);');
-
 	await knex.schema.createTable('actors_photos', (table) => {
 		table.integer('actor_id', 12)
 			.notNullable()
@@ -713,12 +751,20 @@ exports.up = async (knex) => {
 		table.datetime('pinged_at');
 		table.datetime('verified_at');
 
-		table.unique(['actor_id', 'platform', 'handle']);
 		table.unique(['actor_id', 'url']);
 	});
 
 	await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);');
 	await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);');
+	await knex.raw('CREATE UNIQUE INDEX actors_socials_actor_id_platform_handle_lower_unique ON actors_socials (actor_id, platform, lower(handle));');
+
+	await knex.schema.createTable('languages', (table) => {
+		table.string('alpha2')
+			.primary();
+
+		table.text('name');
+		table.text('name_native');
+	});
 
 	await knex.schema.createTable('releases', (table) => {
 		table.increments('id');
@@ -741,6 +787,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -758,6 +805,13 @@ exports.up = async (knex) => {
 		table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
 			.defaultTo('day');
 
+		table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
+			.defaultTo('day');
+
+		table.string('language_alpha2')
+			.references('alpha2')
+			.inTable('languages');
+
 		table.text('description');
 
 		table.integer('duration')
@@ -773,6 +827,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -937,6 +993,8 @@ exports.up = async (knex) => {
 	});
 
 	await knex.schema.createTable('releases_tags', (table) => {
+		table.increments('id');
+
 		table.integer('tag_id', 12)
 			.references('id')
 			.inTable('tags');
@@ -947,9 +1005,13 @@ exports.up = async (knex) => {
 			.inTable('releases')
 			.onDelete('cascade');
 
+		table.integer('actor_id')
+			.references('id')
+			.inTable('actors');
+
 		table.text('original_tag');
 
-		table.unique(['tag_id', 'release_id']);
+		table.unique(['release_id', 'original_tag']);
 		table.index('tag_id');
 		table.index('release_id');
 
@@ -958,12 +1020,7 @@ exports.up = async (knex) => {
 			.defaultTo('scraper');
 	});
 
-	await knex.schema.createTable('releases_search', (table) => {
+	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1));');
-		table.integer('release_id', 16)
-			.references('id')
-			.inTable('releases')
-			.onDelete('cascade');
-	});
 
 	await knex.schema.createTable('movies', (table) => {
 		table.increments('id');
@@ -984,6 +1041,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -1000,6 +1058,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1134,13 +1194,6 @@ exports.up = async (knex) => {
 		table.unique(['movie_id', 'media_id']);
 	});
 
-	await knex.schema.createTable('movies_search', (table) => {
-		table.integer('movie_id', 16)
-			.references('id')
-			.inTable('movies')
-			.onDelete('cascade');
-	});
-
 	await knex.schema.createTable('series', (table) => {
 		table.increments('id');
 
@@ -1159,6 +1212,7 @@ exports.up = async (knex) => {
 		table.text('url', 1000);
 		table.text('title');
 		table.specificType('alt_titles', 'text ARRAY');
+		table.specificType('alt_descriptions', 'text ARRAY');
 		table.text('slug');
 
 		table.timestamp('date');
@@ -1175,6 +1229,8 @@ exports.up = async (knex) => {
 
 		table.text('comment');
 
+		table.json('attributes');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1288,13 +1344,6 @@ exports.up = async (knex) => {
 		table.unique(['serie_id', 'media_id']);
 	});
 
-	await knex.schema.createTable('series_search', (table) => {
-		table.integer('serie_id', 16)
-			.references('id')
-			.inTable('series')
-			.onDelete('cascade');
-	});
-
 	await knex.schema.createTable('chapters', (table) => {
 		table.increments('id');
 
@@ -1316,6 +1365,8 @@ exports.up = async (knex) => {
 		table.text('title');
 		table.text('description');
 
+		table.datetime('date');
+
 		table.integer('created_batch_id', 12)
 			.references('id')
 			.inTable('batches')
@@ -1379,6 +1430,34 @@ exports.up = async (knex) => {
 		table.unique(['tag_id', 'chapter_id']);
 	});
 
+	await knex.schema.createTable('chapters_trailers', (table) => {
+		table.integer('chapter_id')
+			.notNullable()
+			.references('id')
+			.inTable('chapters')
+			.onDelete('cascade');
+
+		table.text('media_id')
+			.notNullable()
+			.references('id')
+			.inTable('media')
+			.onDelete('cascade');
+	});
+
+	await knex.schema.createTable('chapters_teasers', (table) => {
+		table.integer('chapter_id')
+			.notNullable()
+			.references('id')
+			.inTable('chapters')
+			.onDelete('cascade');
+
+		table.text('media_id')
+			.notNullable()
+			.references('id')
+			.inTable('media')
+			.onDelete('cascade');
+	});
+
 	await knex.schema.createTable('users_roles', (table) => {
 		table.string('role')
 			.primary();
@@ -1396,7 +1475,9 @@ exports.up = async (knex) => {
 				{ subject: 'actor', action: 'create' },
 				{ subject: 'actor', action: 'update' },
 				{ subject: 'actor', action: 'delete' },
-				{ plainUrls: true },
+				{ subject: 'actor', action: 'merge' },
+				{ subject: 'sync' },
+				{ subject: 'plainUrls' },
 			]),
 		},
 		{
@@ -1442,6 +1523,8 @@ exports.up = async (knex) => {
 
 		table.specificType('last_ip', 'cidr');
 
+		table.jsonb('settings');
+
 		table.datetime('created_at')
 			.notNullable()
 			.defaultTo(knex.fn.now());
@@ -1660,6 +1743,98 @@ exports.up = async (knex) => {
 			.defaultTo(knex.fn.now());
 	});
 
+	await knex.schema.createTable('feeds', (table) => {
+		table.increments('id');
+
+		table.integer('user_id')
+			.notNullable()
+			.references('id')
+			.inTable('users')
+			.onDelete('cascade');
+
+		table.string('name')
+			.notNullable();
+
+		table.string('slug')
+			.notNullable();
+
+		table.boolean('public');
+		table.boolean('primary');
+
+		table.text('comment');
+		table.json('meta');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_entities', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('entity_id')
+			.notNullable()
+			.references('id')
+			.inTable('entities')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_actors', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('actor_id')
+			.notNullable()
+			.references('id')
+			.inTable('actors')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex.schema.createTable('feeds_tags', (table) => {
+		table.increments('id');
+
+		table.integer('feed_id')
+			.notNullable()
+			.references('id')
+			.inTable('feeds')
+			.onDelete('cascade');
+
+		table.integer('tag_id')
+			.notNullable()
+			.references('id')
+			.inTable('tags')
+			.onDelete('cascade');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.notNullable()
+			.defaultTo(knex.fn.now());
+	});
+
 	await knex.schema.createTable('alerts', (table) => {
 		table.increments('id');
 
@@ -1698,6 +1873,10 @@ exports.up = async (knex) => {
 			.notNullable()
 			.defaultTo(false);
 
+		table.boolean('is_expired')
+			.notNullable()
+			.defaultTo(false);
+
 		table.json('meta');
 		table.text('comment');
 
@@ -1878,6 +2057,31 @@ exports.up = async (knex) => {
 		);
 	});
 
+	await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
+		view.columns('user_id', 'scene_id', 'alert_ids');
+
+		view.as(
+			knex('alerts_scenes')
+				.select(
+					'alerts.user_id',
+					'alerts_scenes.scene_id',
+					knex.raw('array_agg(distinct alerts.id) as alert_ids'),
+					knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
+				)
+				.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
+				.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
+					joinBuilder
+						.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
+						.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
+				})
+				.where('alerts.is_expired', false)
+				.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
+		);
+	});
+
 	await knex.schema.createTable('notifications', (table) => {
 		table.increments('id');
 
@@ -1901,6 +2105,18 @@ exports.up = async (knex) => {
 			.notNullable()
 			.defaultTo(false);
 
+		table.datetime('seen_at');
+
+		table.boolean('notify')
+			.notNullable()
+			.defaultTo(true);
+
+		table.boolean('email')
+			.notNullable()
+			.defaultTo(false);
+
+		table.datetime('emailed_at');
+
 		table.datetime('created_at')
 			.notNullable()
 			.defaultTo(knex.fn.now());
@@ -2177,12 +2393,20 @@ exports.up = async (knex) => {
 			.inTable('entities');
 	});
 
+	await knex.schema.createTable('sync', (table) => {
+		table.increments('id');
+
+		table.string('domain');
+		table.specificType('item_ids', 'integer array');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	});
+
 	// SEARCH AND SORT
 	await knex.raw(`
-		ALTER TABLE releases_search ADD COLUMN document tsvector;
-		ALTER TABLE movies_search ADD COLUMN document tsvector;
-		ALTER TABLE series_search ADD COLUMN document tsvector;
-
 		/* allow scenes without dates to be mixed inbetween scenes with dates */
 		ALTER TABLE releases
 		ADD COLUMN effective_date timestamptz
@@ -2200,19 +2424,12 @@ exports.up = async (knex) => {
 	// INDEXES
 	await knex.raw(`
 		CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id);
-		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
+		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
 
 		CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL;
 		CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL;
-
-		CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
-		CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
-		CREATE INDEX releases_search_index ON releases_search USING GIN (document);
-		CREATE INDEX movies_search_index ON movies_search USING GIN (document);
-		CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
-		CREATE INDEX series_search_index ON series_search USING GIN (document);
 	`);
 
 	// FUNCTIONS
@@ -2275,7 +2492,8 @@ exports.up = async (knex) => {
 				stashes.id as stash_id,
 				COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes,
 				COUNT(DISTINCT stashes_movies)::integer as stashed_movies,
-				COUNT(DISTINCT stashes_actors)::integer as stashed_actors
+				COUNT(DISTINCT stashes_actors)::integer as stashed_actors,
+				MAX(GREATEST(stashes_scenes.created_at, stashes_movies.created_at, stashes_actors.created_at)) as last_stash_at
 			FROM stashes
 			LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id
 			LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id
@@ -2372,6 +2590,8 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS chapters_tags CASCADE;
 		DROP TABLE IF EXISTS chapters_posters CASCADE;
 		DROP TABLE IF EXISTS chapters_photos CASCADE;
+		DROP TABLE IF EXISTS chapters_trailers CASCADE;
+		DROP TABLE IF EXISTS chapters_teasers CASCADE;
 
 		DROP TABLE IF EXISTS banners_tags CASCADE;
 		DROP TABLE IF EXISTS banners CASCADE;
@@ -2381,6 +2601,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 
 		DROP TABLE IF EXISTS batches CASCADE;
 		DROP TABLE IF EXISTS fingerprints_types CASCADE;
+		DROP TABLE IF EXISTS sync CASCADE;
 
 		DROP TABLE IF EXISTS actors_avatars CASCADE;
 		DROP TABLE IF EXISTS actors_photos CASCADE;
@@ -2412,8 +2633,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS sites CASCADE;
 		DROP TABLE IF EXISTS studios CASCADE;
 		DROP TABLE IF EXISTS media_sfw CASCADE;
+		DROP TABLE IF EXISTS media_biometrics CASCADE;
 		DROP TABLE IF EXISTS media CASCADE;
 		DROP TABLE IF EXISTS countries CASCADE;
+		DROP TABLE IF EXISTS languages CASCADE;
 		DROP TABLE IF EXISTS networks CASCADE;
 
 		DROP TABLE IF EXISTS entities_types CASCADE;
@@ -2426,6 +2649,11 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
 		DROP TABLE IF EXISTS stashes_actors CASCADE;
 		DROP TABLE IF EXISTS stashes CASCADE;
 
+		DROP TABLE IF EXISTS feeds_entities CASCADE;
+		DROP TABLE IF EXISTS feeds_actors CASCADE;
+		DROP TABLE IF EXISTS feeds_tags CASCADE;
+		DROP TABLE IF EXISTS feeds CASCADE;
+
 		DROP TABLE IF EXISTS alerts_scenes CASCADE;
 		DROP TABLE IF EXISTS alerts_actors CASCADE;
 		DROP TABLE IF EXISTS alerts_tags CASCADE;

migrations/20260203052449_scene_attributes.js

@@ -1,19 +0,0 @@
-exports.up = async (knex) => {
-	await knex.schema.alterTable('releases', (table) => {
-		table.json('attributes');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.json('attributes');
-	});
-};
-
-exports.down = async (knex) => {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('attributes');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.dropColumn('attributes');
-	});
-};

migrations/20260206034715_raw_releases.js

@@ -1,11 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('alt_descriptions');
-	});
-};

migrations/20260207034922_chapter_details.js

@@ -1,58 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('chapters', (table) => {
-		table.datetime('date');
-	});
-
-	await knex.schema.createTable('chapters_trailers', (table) => {
-		table.integer('chapter_id')
-			.notNullable()
-			.references('id')
-			.inTable('chapters')
-			.onDelete('cascade');
-
-		table.text('media_id')
-			.notNullable()
-			.references('id')
-			.inTable('media')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.createTable('chapters_teasers', (table) => {
-		table.integer('chapter_id')
-			.notNullable()
-			.references('id')
-			.inTable('chapters')
-			.onDelete('cascade');
-
-		table.text('media_id')
-			.notNullable()
-			.references('id')
-			.inTable('media')
-			.onDelete('cascade');
-	});
-
-	await knex.schema.alterTable('entities', (table) => {
-		table.string('name_stylized');
-	});
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('chapters', (table) => {
-		table.dropColumn('date');
-	});
-
-	await knex.schema.alterTable('entities', (table) => {
-		table.dropColumn('name_stylized');
-	});
-
-	await knex.schema.dropTable('chapters_trailers');
-	await knex.schema.dropTable('chapters_teasers');
-
-	await knex.schema.alterTable('movies', (table) => {
-		table.dropColumn('alt_descriptions');
-	});
-};

migrations/20260208044729_random_sfw_improvements.js

@@ -1,28 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createMaterializedView('media_sfw', (view) => {
-		view.as(knex('media').select('id').where('is_sfw', true));
-	});
-
-	await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
-
-	await knex.raw(`
-		CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media_sfw
-			ORDER BY random()
-			LIMIT 1;
-		$$ LANGUAGE sql STABLE;
-	`);
-};
-
-exports.down = async function(knex) {
-	await knex.raw(`
-		CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
-			SELECT id FROM media
-			WHERE is_sfw = true
-			ORDER BY random()
-			LIMIT 1;
-		$$ LANGUAGE sql STABLE;
-	`);
-
-	await knex.schema.dropMaterializedView('media_sfw');
-};

migrations/20260222055254_unique_origin_tags.js

@@ -1,31 +0,0 @@
-exports.up = async (knex) => {
-	// dedupe
-	await knex.raw(`
-		DELETE
-		FROM releases_tags
-		WHERE ctid IN
-		(
-			SELECT ctid
-			FROM(
-				SELECT
-					*,
-					ctid,
-					row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
-				FROM releases_tags
-			)s
-			WHERE row_number >= 2
-		)
-	`);
-
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.increments('id');
-		table.unique(['release_id', 'original_tag']);
-	});
-};
-
-exports.down = async (knex) => {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.dropColumn('id');
-		table.dropUnique(['release_id', 'original_tag']);
-	});
-};

migrations/20260301042453_foot_float.js

@@ -1,23 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('actors', (table) => {
-		table.decimal('foot')
-			.alter();
-	});
-
-	await knex.schema.alterTable('actors_profiles', (table) => {
-		table.decimal('foot')
-			.alter();
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('actors', (table) => {
-		table.integer('foot')
-			.alter();
-	});
-
-	await knex.schema.alterTable('actors_profiles', (table) => {
-		table.integer('foot')
-			.alter();
-	});
-};

migrations/20260302222545_series_alt_descriptions.js

@@ -1,13 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('series', (table) => {
-		table.specificType('alt_descriptions', 'text ARRAY');
-		table.json('attributes');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('series', (table) => {
-		table.dropColumn('alt_descriptions');
-		table.dropColumn('attributes');
-	});
-};

migrations/20260304020542_scene_actor_tags.js

@@ -1,21 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.integer('actor_id')
-			.references('id')
-			.inTable('actors');
-
-		table.dropUnique(['tag_id', 'release_id']);
-	});
-
-	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases_tags', (table) => {
-		table.dropColumn('actor_id');
-
-		table.unique(['tag_id', 'release_id']);
-	});
-
-	await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
-};

migrations/20260402051404_scene_language_production_date_precision.js

@@ -1,27 +0,0 @@
-exports.up = async function(knex) {
-	await knex.schema.createTable('languages', (table) => {
-		table.string('alpha2')
-			.primary();
-
-		table.text('name');
-		table.text('name_native');
-	});
-
-	await knex.schema.alterTable('releases', (table) => {
-		table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
-			.defaultTo('day');
-
-		table.string('language_alpha2')
-			.references('alpha2')
-			.inTable('languages');
-	});
-};
-
-exports.down = async function(knex) {
-	await knex.schema.alterTable('releases', (table) => {
-		table.dropColumn('production_date_precision');
-		table.dropColumn('language_alpha2');
-	});
-
-	await knex.schema.dropTable('languages');
-};

migrations/20260726032348_stashes_archive.js

@@ -0,0 +1,15 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('stashes', (table) => {
+		table.boolean('archived')
+			.notNullable()
+			.defaultTo(false);
+
+		table.check('NOT ("primary" AND archived)', [], 'stashes_not_primary_and_archived');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('stashes', (table) => {
+		table.dropColumn('archived');
+	});
+};

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.252.1",
+    "version": "1.255.9",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {
@@ -76,6 +76,9 @@
         "@graphile-contrib/pg-order-by-related": "^1.0.0",
         "@graphile-contrib/pg-simplify-inflector": "^6.1.0",
         "@graphile/pg-aggregates": "^0.1.1",
+        "@rsc-parser/react-client": "^1.1.2",
+        "@tensorflow/tfjs-node": "^4.22.0",
+        "@vladmandic/human": "^3.3.6",
         "acorn": "^8.11.2",
         "array-equal": "^1.0.2",
         "babel-polyfill": "^6.26.0",
@@ -84,7 +87,6 @@
         "bluebird": "^3.7.2",
         "body-parser": "^1.20.2",
         "bottleneck": "^2.19.5",
-        "canvas": "^2.11.2",
         "casual": "^1.6.2",
         "cheerio": "^1.0.0-rc.12",
         "cli-confirm": "^1.0.1",
@@ -154,7 +156,7 @@
         "tunnel": "0.0.6",
         "ua-parser-js": "^1.0.37",
         "undici": "^7.24.7",
-        "unprint": "^0.19.13",
+        "unprint": "^0.19.20",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.1.3",
         "video.js": "^8.6.1",

seeds/00_tags.js

@@ -95,6 +95,14 @@ const tags = [
 		slug: '69',
 		group: 'position',
 	},
+	{
+		name: 'AI generated',
+		slug: 'ai',
+	},
+	{
+		name: 'AI enhanced',
+		slug: 'aienhanced',
+	},
 	{
 		name: 'airtight',
 		slug: 'airtight',
@@ -114,6 +122,7 @@ const tags = [
 	{
 		name: 'anal',
 		slug: 'anal',
+		rename: 'ass-fucking',
 		description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.',
 		group: 'penetration',
 	},
@@ -160,8 +169,19 @@ const tags = [
 		name: 'ass to other girl\'s mouth',
 		slug: 'atogm',
 		description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".',
+		implies: ['atm'],
 		group: 'oral',
 	},
+	{
+		name: 'ass to pussy',
+		slug: 'atp',
+		description: 'Getting your pussy fucked right after it\'s been in your ass, or [someone else\'s](/tag/atogp).',
+	},
+	{
+		name: 'ass to other girl\'s pussy',
+		slug: 'atogp',
+		description: 'Getting your pussy fucked right after it\'s been in someone else\'s ass.',
+	},
 	{
 		name: 'ass eating',
 		slug: 'ass-eating',
@@ -311,6 +331,10 @@ const tags = [
 		name: 'bondage',
 		slug: 'bondage',
 	},
+	{
+		name: 'bimbo',
+		slug: 'bimbo',
+	},
 	{
 		name: 'braces',
 		slug: 'braces',
@@ -544,8 +568,10 @@ const tags = [
 		slug: 'electric-shock',
 	},
 	{
-		name: 'enhanced boobs',
+		name: 'fake tits',
-		slug: 'enhanced-boobs',
+		slug: 'fake-tits',
+		description: 'Tasteful enhancements or bimbo bolt-ons, you don\'t mind a bit of plastic to get the boobs you\'ve always wanted.',
+		rename: 'enhanced-boobs',
 	},
 	{
 		name: 'enhanced butt',
@@ -647,6 +673,7 @@ const tags = [
 	{
 		name: 'MFF threesome',
 		slug: 'mff',
+		implies: ['threesome'],
 		description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
 		group: 'group',
 	},
@@ -705,6 +732,10 @@ const tags = [
 		name: 'gloryhole',
 		slug: 'gloryhole',
 	},
+	{
+		name: 'female gloryhole',
+		slug: 'female-gloryhole',
+	},
 	{
 		name: 'gonzo',
 		slug: 'gonzo',
@@ -731,6 +762,11 @@ const tags = [
 		slug: 'high-heels',
 		group: 'clothing',
 	},
+	{
+		name: 'hotel',
+		slug: 'hotel',
+		group: 'location',
+	},
 	{
 		name: 'humiliation',
 		slug: 'humiliation',
@@ -826,13 +862,19 @@ const tags = [
 	{
 		name: 'MFM threesome',
 		slug: 'mfm',
-		description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
+		implies: ['threesome'],
+		description: 'Two men fucking one woman, but not eachother. Typically involves a [spitroast](/tag/spitroast), where one guy gets a blowjob and the other fucks her pussy or ass.',
 		group: 'group',
 	},
 	{
 		name: 'spitroast',
 		slug: 'spitroast',
 	},
+	{
+		name: 'anal spitroast',
+		slug: 'anal-spitroast',
+		implies: ['anal', 'spitroast'],
+	},
 	{
 		name: 'military',
 		slug: 'military',
@@ -852,8 +894,9 @@ const tags = [
 		group: 'position',
 	},
 	{
-		name: 'natural boobs',
+		name: 'natural tits',
-		slug: 'natural-boobs',
+		slug: 'natural-tits',
+		rename: 'natural-boobs',
 		group: 'body',
 	},
 	{
@@ -1221,6 +1264,10 @@ const tags = [
 		slug: 'toy-dp',
 		description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
 	},
+	{
+		name: 'finger DP',
+		slug: 'finger-dp',
+	},
 	{
 		name: 'transsexual',
 		slug: 'transsexual',
@@ -1414,6 +1461,10 @@ const tags = [
 		name: 'interview',
 		slug: 'interview',
 	},
+	{
+		name: 'pregnant',
+		slug: 'pregnant',
+	},
 ];
 
 const aliases = [
@@ -1587,6 +1638,10 @@ const aliases = [
 		name: 'mmf',
 		for: 'mfm',
 	},
+	{
+		name: 'threesome mmf',
+		for: 'mfm',
+	},
 	{
 		name: 'fmf',
 		for: 'mff',
@@ -1631,15 +1686,18 @@ const aliases = [
 		name: 'big butts',
 		for: 'big-butt',
 	},
+	{
+		name: 'natural boobs',
+		for: 'natural-tits',
+		slug: 'natural-boobs',
+	},
 	{
 		name: 'big natural tits',
 		for: 'big-boobs',
-		implies: ['natural-boobs'],
 	},
 	{
 		name: 'big natural boobs',
 		for: 'big-boobs',
-		implies: ['natural-boobs'],
 	},
 	{
 		name: 'big tits',
@@ -1657,6 +1715,10 @@ const aliases = [
 		name: 'busty - big boobs',
 		for: 'big-boobs',
 	},
+	{
+		name: 'big breasts',
+		for: 'big-boobs',
+	},
 	{
 		name: 'busty: big beautiful breast',
 		for: 'big-boobs',
@@ -1719,12 +1781,18 @@ const aliases = [
 	},
 	{
 		name: 'fake boobs',
-		for: 'enhanced-boobs',
+		for: 'fake-tits',
+		secondary: true,
+	},
+	{
+		name: 'enhanced boobs',
+		for: 'fake-tits',
+		slug: 'enhanced-boobs',
 		secondary: true,
 	},
 	{
 		name: 'implants',
-		for: 'enhanced-boobs',
+		for: 'fake-tits',
 	},
 	{
 		name: 'boob job',
@@ -1732,7 +1800,7 @@ const aliases = [
 	},
 	{
 		name: 'boobjob',
-		for: 'enhanced-boobs',
+		for: 'fake-tits',
 	},
 	{
 		name: 'brown hair',
@@ -1997,11 +2065,11 @@ const aliases = [
 	},
 	{
 		name: 'enhanced',
-		for: 'enhanced-boobs',
+		for: 'fake-tits',
 	},
 	{
 		name: 'enhanced tits',
-		for: 'enhanced-boobs',
+		for: 'fake-tits',
 	},
 	{
 		name: 'fake butt',
@@ -2044,11 +2112,6 @@ const aliases = [
 		name: 'facial - multiple',
 		for: 'facial',
 	},
-	{
-		name: 'fake tits',
-		for: 'enhanced-boobs',
-		secondary: true,
-	},
 	{
 		name: 'feet sex',
 		for: 'foot-sex',
@@ -2198,11 +2261,7 @@ const aliases = [
 	},
 	{
 		name: 'natural',
-		for: 'natural-boobs',
+		for: 'natural-tits',
-	},
-	{
-		name: 'natural tits',
-		for: 'natural-boobs',
 	},
 	{
 		name: 'natural butt',
@@ -3032,20 +3091,74 @@ const aliases = [
 		name: 't---y f--k',
 		for: 'titty-fucking',
 	},
+	{
+		name: 'thresome',
+		for: 'threesome',
+	},
+	{
+		name: 'fuck',
+		for: 'sex',
+	},
+	{
+		name: 'suck',
+		for: 'blowjob',
+	},
+	{
+		name: 'analfist',
+		for: 'anal-fisting',
+	},
+	{
+		name: 'fivesome',
+		for: 'orgy',
+	},
+	{
+		name: 'fucking machine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuck machine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuckmashine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuck saw',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'dirtytalk',
+		for: 'dirty-talk',
+	},
+	{
+		name: 'stepmom',
+		for: 'family',
+	},
+	{
+		name: 'ass2mouth',
+		for: 'atm',
+	},
+	{
+		name: 'fist',
+		for: 'fisting',
+	},
 ];
 
 const priorities = [ // higher index is higher priority
-	['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
+	['blonde', 'brunette', 'black-hair', 'redhead'],
-	['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
-	['family'],
-	['blowjob', 'deepthroat', 'oil'],
 	['asian', 'black', 'latina', 'white', 'interracial'],
+	['fake-tits', 'natural-tits'],
 	['bts'],
+	['blowjob', 'deepthroat', 'oil'],
+	['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
+	['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
+	['family'],
 	['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'],
 	['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
 	['lesbian', 'rough', 'milf', 'male-focus', 'bdsm', 'oil'],
 	['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
-	['anal', 'bukkake', 'spitroast'],
+	['anal', 'bukkake', 'spitroast', 'anal-spitroast'],
 	['dp', 'dap', 'triple-penetration', 'tap', 'dvp', 'tvp', 'airtight'],
 	['blowbang', 'orgy'],
 	['gangbang'],
@@ -3061,6 +3174,33 @@ const priorities = [ // higher index is higher priority
 exports.seed = (knex) => Promise.resolve()
 	.then(async () => upsert('tags_groups', groups, 'slug', knex))
 	.then(async () => {
+		await Promise.all(tags.map((async (tag) => {
+			if (tag.rename) {
+				await knex('tags')
+					.where((builder) => {
+						builder
+							.where('slug', tag.slug)
+							.orWhere('name', tag.name);
+					})
+					.whereNotNull('alias_for')
+					.delete();
+
+				await knex('tags')
+					.where('slug', tag.rename)
+					.whereNull('alias_for')
+					.update({
+						name: tag.name,
+						slug: tag.slug,
+					});
+			}
+
+			if (tag.delete) {
+				await knex('tags')
+					.where('slug', tag.rename)
+					.delete();
+			}
+		})));
+
 		const groupEntries = await knex('tags_groups').select('*');
 		const groupsMap = groupEntries.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
 
@@ -3108,6 +3248,7 @@ exports.seed = (knex) => Promise.resolve()
 
 			return {
 				name: alias.name,
+				slug: alias.slug, // only used if redirect is required
 				alias_for: tagsMap[alias.for],
 				implied_tag_ids: alias.implies?.map((slug) => tagsMap[slug]),
 				secondary: !!alias.secondary,

seeds/01_networks.js

@@ -239,6 +239,11 @@ const networks = [
 			// scene: false,
 		},
 	},
+	{
+		slug: 'acam',
+		name: 'A-Cam',
+		hasLogo: false,
+	},
 	{
 		slug: 'amateurallure',
 		name: 'Amateur Allure',
@@ -279,6 +284,7 @@ const networks = [
 			areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
 			blockId: 114458,
 			scene: 'https://aziani.com',
+			api: 'https://azianistudios.com/tour_api.php',
 			cdn: 'https://c75c0c3063.mjedge.net',
 		},
 	},
@@ -588,6 +594,11 @@ const networks = [
 			concurrency: 1,
 		},
 	},
+	{
+		slug: 'ksm',
+		name: 'KSM',
+		hasLogo: false,
+	},
 	{
 		slug: 'letsdoeit',
 		name: 'LetsDoeIt',
@@ -616,6 +627,11 @@ const networks = [
 		url: 'https://mamacitaz.com',
 		parent: 'porndoe',
 	},
+	{
+		slug: 'manyvids',
+		name: 'ManyVids',
+		url: 'https://www.manyvids.com',
+	},
 	{
 		slug: 'men',
 		name: 'Men',

seeds/02_sites.js

@@ -6135,6 +6135,12 @@ const sites = [
 			scene: false,
 		},
 	},
+	// JAX SLAYHER
+	{
+		slug: 'jaxslayher',
+		name: 'Jax Slayher',
+		url: 'https://jaxslayher.com',
+	},
 	// JESSE LOADS MONSTER FACIALS
 	{
 		slug: 'jesseloadsmonsterfacials',
@@ -7362,6 +7368,17 @@ const sites = [
 			studioId: 698,
 		},
 	},
+	{
+		name: 'Disciples of Desire',
+		slug: 'disciplesofdesire',
+		url: 'https://disciplesofdesire.com/',
+		parent: 'analvids',
+		independent: true,
+		parameters: {
+			usePornboxUrl: true,
+			studioId: 1214,
+		},
+	},
 	// LETS DOE IT
 	{
 		name: 'The White Boxxx',
@@ -7674,6 +7691,15 @@ const sites = [
 		delete: true,
 		parent: 'mamacitaz',
 	},
+	// MANYVIDS
+	{
+		name: 'nicoledoshi',
+		slug: 'mv.nicoledoshi',
+		url: 'https://nicoledoshi.manyvids.com',
+		hasLogo: false,
+		parent: 'manyvids',
+		delete: true,
+	},
 	// MARISKA X
 	{
 		name: 'MariskaX',
@@ -9440,9 +9466,10 @@ const sites = [
 		parent: 'nubiles',
 	},
 	{
-		slug: 'lilsis',
+		slug: 'realitysis',
-		name: 'Lil Sis',
+		name: 'Reality Sis',
-		url: 'https://lilsis.com',
+		url: 'https://realitysis.com',
+		alias: ['lil sis'],
 		tags: ['family'],
 		parent: 'nubiles',
 	},
@@ -9453,19 +9480,18 @@ const sites = [
 		tags: ['cheating', 'family'],
 		parent: 'nubiles',
 	},
-	{
-		slug: 'realitysis',
-		name: 'Reality Sis',
-		url: 'https://realitysis.com',
-		tags: ['family'],
-		parent: 'nubiles',
-	},
 	{
 		slug: 'shesbreedingmaterial',
 		name: 'She\'s Breeding Material',
 		url: 'https://shesbreedingmaterial.com',
 		parent: 'nubiles',
 	},
+	{
+		slug: 'glowingdesire',
+		name: 'Glowing Desire',
+		url: 'https://glowingdesire.com',
+		parent: 'nubiles',
+	},
 	// PASCALS SUBSLUTS
 	{
 		slug: 'pascalssubsluts',
@@ -10324,6 +10350,12 @@ const sites = [
 		url: 'https://pornplus.com/series/pimpparade',
 		parent: 'pornplus',
 	},
+	{
+		name: 'Penis To Pussy Marketplace',
+		slug: 'p2pmarketplace',
+		url: 'https://pornplus.com/series/p2p-marketplace',
+		parent: 'pornplus',
+	},
 	{
 		name: 'Porn Pros',
 		slug: 'pornprosplus',
@@ -13497,6 +13529,27 @@ const sites = [
 			},
 		},
 	},
+	// SKM
+	{
+		slug: 'cosplayground',
+		name: 'Cosplayground',
+		url: 'https://cosplayground.com',
+		tags: ['cosplay'],
+		parent: 'ksm',
+		independent: true,
+		parameters: {
+			// seems to use same CMS as Aziani
+			api: 'https://api.cosplayground.com/tour_api.php',
+			seriesId: null,
+			areaId: '2bc2444e-a053-40a2-bc4b-ca3618521563',
+			cdn: 'https://c880f36aa8.mjedge.net',
+			entryIdSlug: true,
+			videos: '/videos',
+			scenePath: '/videos',
+			modelPath: '/models',
+			imageType: 'photos',
+		},
+	},
 	// TEEN CORE CLUB
 	{
 		name: 'Analyzed Girls',
@@ -14672,6 +14725,47 @@ const sites = [
 		description: 'BustyOnes.com bringing you the most beautiful big breasts in the world! The hottest women alive showcasing their fantastic tits.',
 		parent: 'twistys',
 	},
+	// A-CAM / VILDE
+	{
+		name: 'Vilde',
+		slug: 'vilde',
+		alias: ['vilde tv', 'johan vilde'],
+		url: 'https://vilde.tv',
+		parent: 'acam',
+		independent: true,
+		parameters: {
+			languageUrl: 'https://vilde.tv/select-language',
+			languageKey: 'select_lang',
+			language: 'English',
+		},
+	},
+	{
+		name: 'Anal Hooked',
+		slug: 'analhooked',
+		url: 'https://analhooked.com',
+		parent: 'acam',
+		tags: ['anal'],
+		independent: true,
+		parameters: {
+			layout: 'hooked',
+			staticUrl: true,
+			languageUrl: 'https://analhooked.com/change-option',
+			languageKey: 'select_language',
+			language: 'english',
+		},
+	},
+	{
+		name: 'Channel Anal',
+		slug: 'channelanal',
+		alias: ['kanal anal'],
+		url: 'https://channelanal.com',
+		parent: 'acam',
+		independent: true,
+		tags: ['anal'],
+		parameters: {
+			layout: 'kanal',
+		},
+	},
 	// VIP SEX VAULT
 	{
 		name: 'Los Consoladores',
@@ -15234,6 +15328,16 @@ sites.reduce((acc, site) => {
 
 /* eslint-disable max-len */
 exports.seed = async (knex) => {
+	sites.reduce((acc, channel) => {
+		if (acc.has(channel.slug)) {
+			console.log('DUPLICATE', channel.slug);
+		} else {
+			acc.add(channel.slug);
+		}
+
+		return acc;
+	}, new Set());
+
 	await Promise.all(sites.map(async (channel) => {
 		if (channel.rename) {
 			await knex('entities')

seeds/04_media.js

@@ -984,6 +984,8 @@ const tagMedia = [
 	['mfm', 'hazel_moore_legalporno', 'Hazel Moore', 'analvids'],
 	['mfm', 7, 'Rose Valerie', 'eurosexparties'],
 	['mfm', 6, 'Honey Gold in "Slut Puppies 12"', 'julesjordan'],
+	['threesome', 'dani_daniels_skin_diamond_hollyrandall_2', 'Dani Daniels, Skin Diamond and Erik Everhard in "Threesome Fun"', 'hollyrandall'],
+	// ['threesome', 'dani_daniels_skin_diamond_hollyrandall', 'Dani Daniels, Skin Diamond and Erik Everhard in "Threesome Fun"', 'hollyrandall'],
 	['natural-boobs', 1, 'Nia Nacci', 'firstclasspov'],
 	['natural-boobs', 4, 'Miela (Marry Queen) in "Pure"', 'femjoy'],
 	['natural-boobs', 7, 'Layla London', 'inthecrack'],
@@ -1144,7 +1146,7 @@ exports.seed = (knex) => Promise.resolve()
 			entity_id: entitiesBySlug[media.entitySlug]?.id,
 		})), 'path', knex);
 
-		const tagIdsBySlug = tags.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
+		const tagIdsBySlug = tags.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.alias_for || tag.id }), {});
 		const mediaIdsByPath = inserted.concat(updated).reduce((acc, item) => ({ ...acc, [item.path]: item.id }), {});
 
 		const tagMediaBySlug = tagMedia.reduce((acc, tagPhoto) => ({

seeds/06_affiliates.js

@@ -807,6 +807,7 @@ const affiliates = [
 		network: 'analvids',
 		parameters: {
 			query: 'aff=BW90MHT1DP', // supported
+			recursive: false,
 		},
 		comment: '50% rev share',
 	},

seeds/08_abilities.js

@@ -0,0 +1,28 @@
+exports.seed = async (knex) => {
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ subject: 'sync' },
+			{ subject: 'plainUrls' },
+		]))
+		.where('role', 'admin');
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ subject: 'plainUrls' },
+		]))
+		.where('role', 'editor');
+};

seeds/09_manyvids.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const unprint = require('unprint');
+const slugify = require('../src/utils/slugify');
+
+const limit = Number(process.env.MV_LIMIT) || 1000;
+const sorting = process.env.MV_SORT || 'followers'; // followers, top, mostActive, newest, trending
+
+async function fetchCreators(page = 1, acc = []) {
+	console.log(`Fetching ${acc.length}/${limit} creators by ${sorting}`);
+
+	// size seems to be capped at 70
+	const res = await unprint.get(`https://api.manyvids.com/search/creators/list?contentPref=1,2,3&sort=${sorting}&size=50&from=${(page - 1) * 50}`);
+
+	if (res.ok && res.data?.creators) {
+		const creators = acc.concat(res.data.creators.map((creator) => ({
+			name: creator.stageName,
+			slug: `mv.${slugify(creator.slug, '')}`,
+			url: `https://www.manyvids.com/Profile/${creator.id}/${creator.slug}/`,
+			emblem: creator.avatar?.url || null,
+		})));
+
+		if (creators.length >= limit) {
+			return creators;
+		}
+
+		return fetchCreators(page + 1, creators);
+	}
+
+	return acc;
+}
+
+exports.seed = async function (knex) {
+	const channels = await fetchCreators();
+
+	const manyvidsNetwork = await knex('entities')
+		.where('slug', 'manyvids')
+		.where('type', 'network')
+		.first();
+
+	if (!manyvidsNetwork) {
+		throw new Error('ManyVids found, did the network migration run?');
+	}
+
+	const result = await knex('entities')
+		.insert(channels.map((channel) => ({
+			name: channel.name,
+			slug: channel.slug,
+			url: channel.url,
+			parent_id: manyvidsNetwork.id,
+			has_logo: false,
+		})))
+		.onConflict(['slug', 'type'])
+		.merge(['name', 'url'])
+		.returning(knex.raw('(xmax = 0) as inserted'));
+
+	console.log(`Done! ${result.filter((row) => row.inserted).length}/${channels.length} ManyVids creators newly added as channels`);
+};

src/actors.js

@@ -30,6 +30,8 @@ const { toBaseReleases } = require('./deep');
 const { associateAvatars, flushOrphanedMedia } = require('./media');
 const { fetchEntitiesBySlug } = require('./entities');
 const { deleteScenes } = require('./releases');
+const { updateActorSearch } = require('./update-search');
+const { setBiometrics } = require('./biometrics');
 
 const actorsCommon = import('../common/actors.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
 const geoCommon = import('../common/geo.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
@@ -212,6 +214,7 @@ function curateActor(actor, withDetails = false, isProfile = false) {
 		name: actor.name,
 		slug: actor.slug,
 		url: actor.url,
+		entryId: actor.entry_id,
 		gender: actor.gender,
 		orientation: actor.orientation,
 		entityId: actor.entity_id,
@@ -608,8 +611,15 @@ async function upsertProfiles(profiles) {
 			media_id: profile.avatarMediaId,
 		}));
 
-		if (avatars.length > 0) {
+		const resolvedAvatars = avatars.filter((avatar) => !!avatar.profile_id);
-			await batchInsert('actors_avatars', avatars, { conflict: false });
+		const unresolvedAvatars = avatars.filter((avatar) => !avatar.profile_id);
+
+		unresolvedAvatars.forEach((avatar) => {
+			logger.warn(`Skipping avatar for actor ${avatar.actor_id} (${avatar.media_id}), no profile could be resolved`);
+		});
+
+		if (resolvedAvatars.length > 0) {
+			await batchInsert('actors_avatars', resolvedAvatars, { conflict: false });
 		}
 	}
 }
@@ -691,7 +701,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 				}
 			}), Promise.reject(new Error()));
 		} catch (error) {
-			console.log(error);
+			// console.log(error);
 
 			if (error.code !== 'PROFILE_NOT_AVAILABLE') {
 				logger.error(`Failed to fetch profile for '${actor.name}': ${error.message}`);
@@ -823,6 +833,11 @@ async function storeProfiles(profiles) {
 
 	await upsertProfiles(profilesWithAvatarIds);
 	await interpolateProfiles(actorIds);
+
+	setBiometrics(Array.from(new Set(profiles.map((profile) => profile.actorId)))).catch((error) => {
+		// biometrics are courtesy, don't await
+		logger.error(`Biometrics failed: ${error.message}`);
+	});
 }
 
 async function scrapeActors(argNames) {
@@ -912,7 +927,7 @@ async function getOrCreateActors(baseActors, batchId) {
 		.whereRaw(`
 			actors.slug = base_actors.slug
 			AND actors.entity_id IS NULL
-			AND NOT base_actors.collision_likely
+			AND (NOT base_actors.collision_likely OR actors.allow_global_match)
 		`)
 		.orWhereRaw(`
 			actors.slug = base_actors.slug
@@ -973,6 +988,7 @@ async function getOrCreateActors(baseActors, batchId) {
 				...actor,
 				actorId,
 				update: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
+				profileId: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
 			};
 		})
 		.filter((actor) => !!actor.actorId)
@@ -980,7 +996,8 @@ async function getOrCreateActors(baseActors, batchId) {
 
 	await storeProfiles(newActorProfiles);
 
-	if (Array.isArray(newActors)) {
+	if (Array.isArray(newActors) && newActors.length > 0) {
+		await updateActorSearch(newActors.map((actor) => actor.id));
 		return newActors.concat(existingActors);
 	}
 

src/alerts.js

@@ -1,283 +0,0 @@
-'use strict';
-
-const escapeRegexp = require('escape-string-regexp');
-
-const argv = require('./argv');
-const logger = require('./logger')(__filename);
-const knex = require('./knex');
-const { indexApi } = require('./manticore');
-const bulkInsert = require('./utils/bulk-insert');
-const { HttpError } = require('./errors');
-
-async function addAlert(alert, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
-	}
-
-	if ((!alert.actors || alert.actors.length === 0) && (!alert.tags || alert.tags.length === 0) && (!alert.entities || alert.entities.length === 0) && (!alert.matches || alert.matches.length === 0)) {
-		throw new HttpError('Alert must contain at least one actor, tag or entity', 400);
-	}
-
-	if (alert.matches?.some((match) => !match.property || !match.expression)) {
-		throw new HttpError('Match must define a property and an expression', 400);
-	}
-
-	const [{ id: alertId }] = await knex('alerts')
-		.insert({
-			user_id: sessionUser.id,
-			notify: alert.notify,
-			email: alert.email,
-			all: alert.all,
-		})
-		.returning('id');
-
-	await Promise.all([
-		alert.actors?.length > 0 && bulkInsert('alerts_actors', alert.actors.map((actorId) => ({
-			alert_id: alertId,
-			actor_id: actorId,
-		})), false),
-		alert.tags?.length > 0 && bulkInsert('alerts_tags', alert.tags.map((tagId) => ({
-			alert_id: alertId,
-			tag_id: tagId,
-		})), false),
-		alert.matches?.length > 0 && bulkInsert('alerts_matches', alert.matches.map((match) => ({
-			alert_id: alertId,
-			property: match.property,
-			expression: /\/.*\//.test(match.expression)
-				? match.expression.slice(1, -1)
-				: escapeRegexp(match.expression),
-		})), false),
-		alert.stashes?.length > 0 && bulkInsert('alerts_stashes', alert.stashes.map((stashId) => ({
-			alert_id: alertId,
-			stash_id: stashId,
-		})), false),
-		alert.entities && bulkInsert('alerts_entities', alert.entities.map((entityId) => ({
-			alert_id: alertId,
-			entity_id: entityId,
-		})).slice(0, alert.all ? 1 : Infinity), false), // one scene can never match multiple entities in AND mode
-	]);
-
-	return alertId;
-}
-
-async function removeAlert(alertId) {
-	await knex('alerts').where('id', alertId).delete();
-}
-
-async function notify(scenes) {
-	if (argv.showcased === false) {
-		logger.info(`Skipping notify for ${scenes.length} scenes because showcase flag is disabled.`);
-		return [];
-	}
-
-	const sceneIds = scenes.map((scene) => scene.id);
-
-	const [
-		releasesActors,
-		releasesTags,
-		rawAlerts,
-		alertsActors,
-		alertsTags,
-		alertsEntities,
-		alertsMatches,
-		alertsStashes,
-	] = await Promise.all([
-		knex('releases_actors').whereIn('release_id', sceneIds),
-		knex('releases_tags').whereIn('release_id', sceneIds),
-		knex('alerts'),
-		knex('alerts_actors'),
-		knex('alerts_tags'),
-		knex('alerts_entities'),
-		knex('alerts_matches'),
-		knex('alerts_stashes'),
-	]);
-
-	const actorIdsByReleaseId = releasesActors.reduce((acc, releaseActor) => {
-		if (!acc[releaseActor.release_id]) {
-			acc[releaseActor.release_id] = [];
-		}
-
-		acc[releaseActor.release_id].push(releaseActor.actor_id);
-
-		return acc;
-	}, {});
-
-	const tagIdsByReleaseId = releasesTags.reduce((acc, releaseTag) => {
-		if (!acc[releaseTag.release_id]) {
-			acc[releaseTag.release_id] = [];
-		}
-
-		acc[releaseTag.release_id].push(releaseTag.tag_id);
-
-		return acc;
-	}, {});
-
-	const alertsActorsByAlertId = alertsActors.reduce((acc, alertActor) => { if (!acc[alertActor.alert_id]) { acc[alertActor.alert_id] = []; } acc[alertActor.alert_id].push(alertActor.actor_id); return acc; }, {});
-	const alertsTagsByAlertId = alertsTags.reduce((acc, alertTag) => { if (!acc[alertTag.alert_id]) { acc[alertTag.alert_id] = []; } acc[alertTag.alert_id].push(alertTag.tag_id); return acc; }, {});
-	const alertsEntitiesByAlertId = alertsEntities.reduce((acc, alertEntity) => { if (!acc[alertEntity.alert_id]) { acc[alertEntity.alert_id] = []; } acc[alertEntity.alert_id].push(alertEntity.entity_id); return acc; }, {});
-	const alertsStashesByAlertId = alertsStashes.reduce((acc, alertStash) => { if (!acc[alertStash.alert_id]) { acc[alertStash.alert_id] = []; } acc[alertStash.alert_id].push(alertStash.stash_id); return acc; }, {});
-
-	const alertsMatchesByAlertId = alertsMatches.reduce((acc, alertMatch) => {
-		if (!acc[alertMatch.alert_id]) {
-			acc[alertMatch.alert_id] = [];
-		}
-
-		acc[alertMatch.alert_id].push({
-			property: alertMatch.property,
-			expression: new RegExp(alertMatch.expression, 'ui'),
-		});
-
-		return acc;
-	}, {});
-
-	const alerts = rawAlerts.map((alert) => ({
-		id: alert.id,
-		userId: alert.user_id,
-		notify: alert.notify,
-		email: alert.email,
-		all: alert.all,
-		allActors: alert.all_actors,
-		allEntities: alert.all_entities,
-		allTags: alert.all_tags,
-		allMatches: alert.all_matches,
-		actors: alertsActorsByAlertId[alert.id] || [],
-		tags: alertsTagsByAlertId[alert.id] || [],
-		entities: alertsEntitiesByAlertId[alert.id] || [],
-		matches: alertsMatchesByAlertId[alert.id] || [],
-		stashes: alertsStashesByAlertId[alert.id] || [],
-	}));
-
-	const curatedScenes = scenes.map((scene) => ({
-		id: scene.id,
-		title: scene.title,
-		description: scene.description,
-		actorIds: actorIdsByReleaseId[scene.id] || [],
-		tagIds: tagIdsByReleaseId[scene.id] || [],
-		entityId: scene.entity.id,
-		parentEntityId: scene.entity.parent?.id,
-	}));
-
-	const triggers = alerts.flatMap((alert) => {
-		const alertScenes = curatedScenes.filter((scene) => {
-			if (alert.all) {
-				if (alert.actors.length > 0 && !alert.actors[alert.allActors ? 'every' : 'some']((actorId) => scene.actorIds.includes(actorId))) {
-					return false;
-				}
-
-				if (alert.tags.length > 0 && !alert.tags[alert.allTags ? 'every' : 'some']((tagId) => scene.tagIds.includes(tagId))) {
-					return false;
-				}
-
-				// multiple entities can only be matched in OR mode
-				if (alert.entities.length > 0 && !alert.entities.some((alertEntityId) => alertEntityId === scene.entityId || alertEntityId === scene.parentEntityId)) {
-					return false;
-				}
-
-				if (alert.matches.length > 0 && !alert.matches[alert.allMatches ? 'every' : 'some']((match) => match.expression.test(scene[match.property]))) {
-					return false;
-				}
-
-				return true;
-			}
-
-			if (alert.matches.length > 0 && alert.actors[alert.allActors ? 'every' : 'some']((actorId) => scene.actorIds.includes(actorId))) {
-				return true;
-			}
-
-			if (alert.tags.length > 0 && alert.tags[alert.allTags ? 'every' : 'some']((tagId) => scene.tagIds.includes(tagId))) {
-				return true;
-			}
-
-			// multiple entities can only be matched in OR mode
-			if (alert.entities.length > 0 && alert.entities.some((alertEntityId) => alertEntityId === scene.entityId || alertEntityId === scene.parentEntityId)) {
-				return true;
-			}
-
-			if (alert.matches.length > 0 && alert.matches[alert.allMatches ? 'every' : 'some']((match) => match.expression.test(scene[match.property]))) {
-				return true;
-			}
-
-			return false;
-		});
-
-		return alertScenes.map((scene) => ({
-			sceneId: scene.id,
-			alert,
-		}));
-	});
-
-	const notifications = Object.values(Object.fromEntries(triggers // prevent multiple notifications for the same scene
-		.filter((trigger) => trigger.alert.notify)
-		.map((trigger) => [`${trigger.alert.userId}:${trigger.sceneId}`, trigger])))
-		.map((trigger) => ({
-			user_id: trigger.alert.userId,
-			alert_id: trigger.alert.id,
-			scene_id: trigger.sceneId,
-		}));
-
-	const uniqueStashes = Object.values(Object.fromEntries(triggers.flatMap((trigger) => trigger.alert.stashes.map((stashId) => ({
-		stashId,
-		sceneId: trigger.sceneId,
-		userId: trigger.alert.userId,
-	}))).map((stash) => [`${stash.stashId}:${stash.sceneId}`, stash])));
-
-	const stashEntries = uniqueStashes.map((stash) => ({
-		scene_id: stash.sceneId,
-		stash_id: stash.stashId,
-	}));
-
-	const [stashed] = await Promise.all([
-		bulkInsert('stashes_scenes', stashEntries, false),
-		bulkInsert('notifications', notifications, false),
-	]);
-
-	// we need created_at from the databased, but user_id is not returned. it's easier to query it than to try and merge it with the input data
-	const stashedEntries = await knex('stashes_scenes')
-		.select('stashes_scenes.*', 'stashes.user_id')
-		.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
-		.whereIn('stashes_scenes.id', stashed.map((stash) => stash.id));
-
-	const docs = stashedEntries.map((stash) => ({
-		replace: {
-			index: 'scenes_stashed',
-			id: stash.id,
-			doc: {
-				scene_id: stash.scene_id,
-				user_id: stash.user_id,
-				stash_id: stash.stash_id,
-				created_at: Math.round(stash.created_at.getTime() / 1000),
-			},
-		},
-	}));
-
-	if (docs.length > 0) {
-		await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-	}
-
-	return triggers;
-}
-
-async function updateNotification(notificationId, notification, sessionUser) {
-	await knex('notifications')
-		.where('user_id', sessionUser.id)
-		.where('id', notificationId)
-		.update({
-			seen: notification.seen,
-		});
-}
-
-async function updateNotifications(notification, sessionUser) {
-	await knex('notifications')
-		.where('user_id', sessionUser.id)
-		.update({
-			seen: notification.seen,
-		});
-}
-
-module.exports = {
-	addAlert,
-	removeAlert,
-	notify,
-	updateNotification,
-	updateNotifications,
-};

src/app.js

@@ -252,7 +252,12 @@ async function init() {
 			const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
 			const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
 
-			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
+			const allScenes = [...storedScenes, ...storedMovieScenes];
+
+			await associateMovieScenes(storedMovies, allScenes);
+
+			await updateSceneSearch(allScenes.map((scene) => scene.id));
+			await updateMovieSearch(storedMovies.map((movie) => movie.id));
 		}
 
 		logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() }))

src/biometrics.js

@@ -0,0 +1,213 @@
+'use strict';
+
+const config = require('config');
+const path = require('path');
+const fsPromises = require('fs').promises;
+const Human = require('@vladmandic/human').default;
+
+const knex = require('./knex');
+const logger = require('./logger')(__filename);
+
+const humanConfig = {
+	modelBasePath: `file://${path.join(__dirname, '../node_modules/@vladmandic/human/models')}/`,
+	backend: 'tensorflow', // uses tfjs-node under the hood
+	face: {
+		enabled: true,
+		detector: { rotation: false },
+		mesh: { enabled: false },
+		iris: { enabled: false },
+		description: { enabled: true },
+		emotion: { enabled: true },
+	},
+	body: { enabled: false },
+	hand: { enabled: false },
+	gesture: { enabled: false },
+};
+
+let humanInstance = null;
+const nsPerSec = 1e9;
+const remainingTimeWindow = 30; // number of recent entries used to estimate time remaining
+
+async function getHumanInstance() {
+	if (!humanInstance) {
+		humanInstance = new Human(humanConfig);
+
+		await humanInstance.load();
+		await humanInstance.warmup();
+	}
+
+	return humanInstance;
+}
+
+async function getImageBuffer(mediaEntry) {
+	if (mediaEntry.is_s3) {
+		if (!config.s3.enabled) {
+			logger.verbose(`Skipping biometrics for ${mediaEntry.media_id}, S3 not enabled`);
+			return null;
+		}
+
+		const res = await fetch(`https://${config.s3.bucket}/${mediaEntry.path}`);
+
+		if (res.ok) {
+			return Buffer.from(await res.arrayBuffer());
+		}
+
+		return null;
+	}
+
+	try {
+		const buffer = await fsPromises.readFile(path.join(config.media.path, mediaEntry.path));
+
+		return buffer;
+	} catch (_error) {
+		return null;
+	}
+}
+
+async function getBiometrics(avatarEntry) {
+	if (!avatarEntry) {
+		return null;
+	}
+
+	const human = await getHumanInstance();
+	const buffer = await getImageBuffer(avatarEntry);
+
+	if (!buffer) {
+		return null;
+	}
+
+	let tensor;
+
+	try {
+		tensor = human.tf.node.decodeImage(buffer, 3);
+		const result = await human.detect(tensor);
+
+		return result;
+	} catch (error) {
+		logger.error(`Failed to decode or detect ${avatarEntry.media_id}: ${error.message}`);
+
+		return null;
+	} finally {
+		tensor?.dispose();
+	}
+}
+
+function getAvatarSource(actorIds, includePhotos) {
+	if (includePhotos) {
+		return knex('actors_avatars')
+			.select('media_id')
+			.modify((builder) => {
+				if (actorIds) {
+					builder.whereIn('actor_id', actorIds);
+				}
+			});
+	}
+
+	return knex('actors')
+		.select('avatar_media_id as media_id')
+		.whereNotNull('avatar_media_id')
+		.modify((builder) => {
+			if (actorIds) {
+				builder.whereIn('id', actorIds);
+			}
+		});
+}
+
+async function setBiometrics(actorIds, shouldUpdate = false, includePhotos = false) {
+	if (!config.media.biometrics.enabled) {
+		return;
+	}
+
+	const startTime = process.hrtime();
+	const recentDurationsMs = [];
+
+	const avatarSource = getAvatarSource(actorIds, includePhotos);
+
+	const avatarEntries = await knex('media')
+		.select(
+			'media.id as media_id',
+			'media.path',
+			'media.is_s3',
+			knex.raw('coalesce(json_agg(media_biometrics.biometrics) filter (where media_biometrics.id is not null), \'[]\') as biometrics'),
+		)
+		.leftJoin('media_biometrics', 'media_biometrics.media_id', 'media.id')
+		.whereIn('media.id', avatarSource)
+		.groupBy('media.id', 'media.path', 'media.is_s3');
+
+	await avatarEntries.reduce(async (chain, avatarEntry, avatarIndex) => {
+		await chain;
+
+		if (avatarEntry.biometrics.length > 0 && !shouldUpdate) {
+			logger.verbose(`Skipping biometrics for ${avatarEntry.media_id}, already present`);
+			return;
+		}
+
+		const entryStartTime = process.hrtime();
+
+		const biometrics = await getBiometrics(avatarEntry);
+
+		if (!biometrics) {
+			return;
+		}
+
+		const curatedBiometrics = biometrics.face.map((face) => ({
+			biometrics: {
+				age: face.age,
+				gender: face.gender,
+				genderScore: face.genderScore,
+				emotion: face.emotion,
+				box: face.box,
+				score: face.score,
+				...Object.fromEntries(Object.entries(face.annotations).map(([key, annotation]) => [key, annotation.filter(Boolean)[0] || null])),
+			},
+			embedding: face.embedding,
+		}));
+
+		if (curatedBiometrics.length === 0) {
+			return;
+		}
+
+		if (shouldUpdate) {
+			await knex('media_biometrics')
+				.where('media_id', avatarEntry.media_id)
+				.delete();
+		}
+
+		await knex('media_biometrics')
+			.insert(curatedBiometrics.map((face, index) => ({
+				media_id: avatarEntry.media_id,
+				face_index: index,
+				width: biometrics.width,
+				height: biometrics.height,
+				biometrics: JSON.stringify(face.biometrics),
+				embedding: knex.raw('?::vector', [JSON.stringify(face.embedding)]),
+				updated_at: knex.raw('now()'),
+			})))
+			.onConflict(['media_id', 'face_index'])
+			.ignore();
+
+		const [entrySec, entryNs] = process.hrtime(entryStartTime);
+		const entryMs = (entrySec * nsPerSec + entryNs) / 1e6;
+
+		recentDurationsMs.push(entryMs);
+
+		if (recentDurationsMs.length > remainingTimeWindow) {
+			recentDurationsMs.shift();
+		}
+
+		const avgMsPerEntry = recentDurationsMs.reduce((sum, ms) => sum + ms, 0) / recentDurationsMs.length;
+		const remainingMs = avgMsPerEntry * (avatarEntries.length - (avatarIndex + 1));
+		const remainingMinutes = (remainingMs / 1000 / 60).toFixed(1);
+
+		logger.info(`Set biometrics for ${avatarEntry.media_id} (${avatarIndex + 1}/${avatarEntries.length}, ~${remainingMinutes}m remaining)`);
+	}, Promise.resolve());
+
+	const diffTime = process.hrtime(startTime);
+	const totalMs = (diffTime[0] * nsPerSec + diffTime[1]) / 1e6;
+
+	logger.info(`Biometrics done in ${(totalMs / 1000).toFixed(1)}s for ${actorIds?.length || 'all'} actors and ${avatarEntries.length} avatars`);
+}
+
+module.exports = {
+	setBiometrics,
+};

src/media.js

@@ -28,7 +28,6 @@ const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 const { fetchEntityReleaseIds } = require('./entity-releases');
 
-// const pipeline = util.promisify(stream.pipeline);
 const streamQueue = taskQueue();
 
 const s3 = new S3Client({
@@ -516,16 +515,6 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
 			writeLazy(image, lazypath, info),
 		]);
 
-		/*
-		if (isProcessed) {
-			// file already stored, remove temporary file
-			await fsPromises.unlink(media.file.path);
-		} else {
-			// image not processed, simply move temporary file to final location
-			await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
-		}
-		*/
-
 		await fsPromises.unlink(media.file.path);
 
 		if (config.s3.enabled) {
@@ -731,31 +720,31 @@ async function fetchSource(source, baseMedia) {
 	}
 
 	async function attempt(attempts = 1) {
-		const hasher = new blake2.Hash('blake2b', { digestLength: 24 });
+		const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
-		let hasherReady = true;
+		let tempFileTarget;
-		hasher.setEncoding('hex');
+
+		const hasher = blake2.createHash('blake2b', { digestLength: 24 });
+
+		let size = 0;
+
+		const hashStream = new stream.Transform({
+			transform(streamChunk, _encoding, callback) {
+				size += streamChunk.length;
+				hasher.update(streamChunk);
+				this.push(streamChunk);
+				callback();
+			},
+		});
 
 		try {
-			const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
+			tempFileTarget = fs.createWriteStream(tempFilePath);
-			const tempFileTarget = fs.createWriteStream(tempFilePath);
-			const hashStream = new stream.PassThrough();
-			let size = 0;
-
-			hashStream.on('data', (streamChunk) => {
-				size += streamChunk.length;
-
-				if (hasherReady) {
-					hasher.write(streamChunk);
-				}
-			});
 
 			const { mimetype } = source.stream
 				? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
 				: await fetchHttpSource(source, tempFileTarget, hashStream);
 
-			hasher.end();
+			const hash = hasher.digest('hex');
 
-			const hash = hasher.read();
 			const [type, subtype] = mimetype.split('/');
 			const extension = mime.getExtension(mimetype);
 
@@ -778,8 +767,10 @@ async function fetchSource(source, baseMedia) {
 				},
 			};
 		} catch (error) {
-			hasherReady = false;
+			// hasherReady = false;
-			hasher.end();
+			// hasher.end();
+			hashStream.destroy();
+			tempFileTarget?.destroy();
 
 			if (error.code !== 'VERIFY_TYPE') {
 				logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);

src/scrapers/acam.js

@@ -0,0 +1,250 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const slugify = require('../utils/slugify');
+
+function extractEntryId(posters) {
+	try {
+		const poster = [].concat(posters).filter(Boolean)[0];
+		return slugify(new URL(poster.src || poster).pathname.match(/\/images\/(.*?)\.[a-z]{3,4}/i)?.[1]?.replace(/smak.*/i, ''), '');
+	} catch (error) {
+		return null;
+	}
+}
+
+function extractTags(title, titleComment) {
+	if (!title && !titleComment) {
+		return [];
+	}
+
+	if (titleComment?.includes('<i>')) {
+		const tagsMatch = titleComment.match(/<i>(.*?)<\/i>/)?.[1];
+
+		if (tagsMatch) {
+			return tagsMatch.split('-').map((tag) => tag.trim().toLowerCase());
+		}
+	}
+
+	const firstTagIndex = title.match(/[A-Z]{2}/)?.index;
+
+	if (firstTagIndex) {
+		const tagSection = title
+			.slice(firstTagIndex)
+			.match(/([A-Z0-9\s]{2,})/g);
+
+		if (tagSection) {
+			return tagSection
+				.map((tag) => tag.trim().toLowerCase())
+				.filter(Boolean) || [];
+		}
+	}
+
+	return [];
+}
+
+// derived photo is usually uncensored and preferred as poster, but not guaranteed to exist, so fall back to original image
+function getPhotos(poster) {
+	const photo = poster?.replace(/(s[ma]{2}kprov\d*)|([._]preview)/i, ''); // sic
+
+	if (photo === poster) {
+		return {
+			poster: {
+				src: poster,
+				verifyType: 'image',
+			},
+			photos: [],
+		};
+	}
+
+	return {
+		poster: [{
+			src: photo,
+			verifyType: 'image',
+		}, {
+			src: poster,
+			verifyType: 'image',
+		}],
+		photos: [{
+			src: poster,
+			verifyType: 'image',
+		}],
+	};
+}
+
+function scrapeAll(scenes, channel, parameters) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		// Vilde URLs are temporary tokens for some reason, seem to be handled entirely back-end
+		const url = query.url('a[href*="/show-video"]');
+
+		release.token = new URL(url).pathname.match(/\/show-video\/([a-z0-9]+)/)?.[1];
+		release.forceDeep = true;
+
+		release.title = query.content('a h5, .product-content p, .video_text');
+		release.tags = extractTags(release.title, query.content('//a/comment()'));
+
+		const { poster, photos } = getPhotos(query.img('img[src*="/videos/images"], img[src*="/uploads/images"]'));
+
+		release.poster = poster;
+		release.photos = photos;
+
+		if (parameters.staticUrl) {
+			release.url = url;
+			release.entryId = release.token;
+		} else {
+			release.entryId = extractEntryId(release.poster);
+		}
+
+		return release;
+	});
+}
+
+async function setLanguage(parameters) {
+	if (parameters.languageUrl) {
+		const langRes = await unprint.post(parameters.languageUrl, {
+			[parameters.languageKey || 'select_language']: parameters.language || 'english',
+		}, {
+			form: true,
+		});
+
+		return langRes.cookies;
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.post(`${channel.origin}/pagination`, {
+		i: page,
+		status: true,
+	}, {
+		selectAll: '.movi-area',
+		form: true,
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestHooked(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.get(`${channel.origin}/Welcome/index/${(page - 1) * 9}`, {
+		selectAll: '.product-main',
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestKanal(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.post(`${channel.origin}/pagination`, {
+		k: page,
+		hidden_page_no: page - 1,
+		status: true,
+	}, {
+		selectAll: '.video_bx',
+		form: true,
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+function scrapeScene({ query }, { url, baseRelease, parameters }) {
+	const release = {};
+
+	// URL is temporary token
+	if (!query.exists('.login-sec.for-browser, .video-description, .video_co_title')) {
+		// URL likely expired, still returns 200
+		return null;
+	}
+
+	if (query.exists('.video-description')) {
+		const descriptions = query.contents('.video-description p').filter(Boolean);
+
+		release.title = descriptions[0];
+		release.description = descriptions.slice(1).join(' ') || null;
+	} else {
+		release.title = query.content('.login-sec.for-browser h3, .video_co_title h3');
+		release.description = query.contents('.login-sec.for-browser h3 ~ *').join(' ') || null;
+	}
+
+	release.tags = extractTags(release.title);
+
+	const { poster, photos } = getPhotos(query.poster('.play_video_cont video'));
+
+	release.poster = poster;
+	release.photos = photos;
+
+	release.trailer = query.all('.play_video_cont source')
+		.map((videoEl) => ({
+			src: unprint.query.url(videoEl, null, { attribute: 'src' }),
+			quality: unprint.query.number(videoEl, null, { attribute: 'size' }),
+			referer: url,
+		}))
+		.toSorted((videoA, videoB) => videoB.quality - videoA.quality);
+
+	if (parameters.staticUrl) {
+		release.url = url;
+		release.entryId = baseRelease?.token || new URL(url).pathname.match(/\/show-video\/([a-z0-9]+)/)?.[1];
+	} else {
+		release.entryId = extractEntryId(release.poster);
+	}
+
+	return release;
+}
+
+async function fetchScene(baseUrl, entity, baseRelease, { parameters }) {
+	const url = baseUrl || (baseRelease?.token && `${entity.origin}/show-video/${baseRelease.token}`) || null;
+
+	if (!url) {
+		return null;
+	}
+
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.get(url, {
+		headers: {
+			'accept-language': 'en-US,en',
+		},
+		cookies,
+	});
+
+	if (res.ok || res.status === 500) { // Anal Hooked returns 500 for valid scene pages
+		return scrapeScene(res.context, { url, baseRelease, parameters });
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	hooked: {
+		fetchLatest: fetchLatestHooked,
+		fetchScene,
+	},
+	kanal: {
+		fetchLatest: fetchLatestKanal,
+		fetchScene,
+	},
+};

src/scrapers/actors.js

@@ -23,6 +23,7 @@ const hitzefrei = require('./hitzefrei');
 const hookuphotshot = require('./hookuphotshot');
 const hush = require('./hush');
 const inthecrack = require('./inthecrack');
+const jaxslayher = require('./jaxslayher');
 const julesjordan = require('./julesjordan');
 const karups = require('./karups');
 const kellymadison = require('./kellymadison');
@@ -33,6 +34,7 @@ const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
 const bluedonkeymedia = require('./bluedonkeymedia');
+const manyvids = require('./manyvids');
 const mikeadriano = require('./mikeadriano');
 const aylo = require('./aylo');
 const missax = require('./missax');
@@ -205,14 +207,6 @@ module.exports = {
 	jerkaoke: modelmedia,
 	modelmediaasia: modelmedia,
 	delphine: modelmedia,
-	// etc
-	'18vr': badoink,
-	theflourishxxx: theflourish,
-	pierrewoodman,
-	exploitedx, // only from known URL that will specify site
-	fullpornnetwork,
-	adultempire,
-	allherluv: missax,
 	// americanpornstar,
 	angelogodshackoriginal,
 	babevr: badoink,
@@ -230,6 +224,7 @@ module.exports = {
 	hitzefrei,
 	hookuphotshot,
 	inthecrack,
+	jaxslayher,
 	karups,
 	boyfun: karups,
 	kellymadison,
@@ -281,4 +276,14 @@ module.exports = {
 	slayed: vixen,
 	wifey: vixen,
 	vrcosplayx: badoink,
+	// etc
+	'18vr': badoink,
+	theflourishxxx: theflourish,
+	cosplayground: aziani,
+	pierrewoodman,
+	exploitedx, // only from known URL that will specify site
+	fullpornnetwork,
+	adultempire,
+	allherluv: missax,
+	manyvids,
 };

src/scrapers/aylo.js

@@ -1,15 +1,10 @@
 'use strict';
 
 /* eslint-disable newline-per-chained-call */
-const Promise = require('bluebird');
-const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
-const cookie = require('cookie');
 const moment = require('moment');
-// const unprint = require('unprint');
+const unprint = require('unprint');
 
-const qu = require('../utils/qu');
 const slugify = require('../utils/slugify');
-const http = require('../utils/http');
 const { inchesToCm, lbsToKg } = require('../utils/convert');
 
 function getBasePath(parameters, channel, path = '/scene') {
@@ -126,6 +121,119 @@ async function scrapeLatest(items, site, filterChannel, options) {
 	};
 }
 
+function getUrl(site) {
+	const { searchParams, pathname } = new URL(site.url);
+
+	// if (search.match(/\?site=\d+/)) {
+	if (searchParams.has('site') || /\/site\/\d+/.test(pathname)) {
+		return site.url;
+	}
+
+	if (site.parameters?.native) {
+		return `${site.url}/scenes`;
+	}
+
+	if (site.parameters?.extract) {
+		return `${site.url}/scenes`;
+	}
+
+	if (site.parameters?.siteId) {
+		return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
+	}
+
+	throw new Error(`Aylo site '${site.name}' (${site.url}) not supported`);
+}
+
+async function getSession(site, _parameters, url) {
+	// if (site.slug === 'aylo' || site.parameters?.parentSession === false) {
+	if (site.slug === 'aylo') {
+		// most MG sites have a parent network to acquire a session from, don't try to acquire session from mindgeek.com for independent channels
+		return null;
+	}
+
+	const sessionUrl = site.parameters?.siteId && !(site.parameters?.native || site.parameters?.childSession || site.parent?.parameters?.childSession || site.parameters?.parentSession === false)
+		? site.parent.url
+		: (url || site.url);
+
+	const res = await unprint.get(sessionUrl, {
+		headers: {
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+			Connection: 'keep-alive',
+		},
+	});
+
+	if (res.status === 200) {
+		const instanceToken = res.cookies.instance_token;
+
+		if (instanceToken) {
+			return { instanceToken };
+		}
+	}
+
+	throw new Error(`Failed to acquire Aylo session (${res.statusCode})`);
+}
+
+async function fetchLatest(site, page = 1, options) {
+	const url = getUrl(site);
+	const { searchParams, pathname } = new URL(url);
+	const siteId = searchParams.get('site') || Number(pathname.match(/\/site\/(\d+)\//)?.[1]);
+
+	if (!siteId && !site.parameters?.native && !site.parameters?.extract) {
+		return null;
+	}
+
+	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
+		? options.beforeNetwork
+		: await getSession(site, options.parameters, url);
+
+	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+	const limit = 24;
+	const apiUrl = site.parameters?.native || site.parameters?.extract
+		? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
+		: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
+
+	const res = await unprint.get(apiUrl, {
+		interval: options.parameters.interval,
+		concurrency: options.parameters.concurrency,
+		headers: {
+			Instance: instanceToken,
+			Origin: site.url,
+			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
+	});
+
+	if (res.status === 200 && res.data.result) {
+		return scrapeLatest(res.data.result, site, false, options);
+	}
+
+	return res.status;
+}
+
+async function fetchUpcoming(site, _page, options) {
+	const url = getUrl(site);
+	const { instanceToken } = await getSession(site, options.parameters);
+
+	const apiUrl = 'https://site-api.project1service.com/v2/upcoming-releases';
+
+	const res = await unprint.get(apiUrl, {
+		interval: options.parameters.interval,
+		concurrency: options.parameters.concurrency,
+		headers: {
+			Instance: instanceToken,
+			Origin: site.url,
+			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
+	});
+
+	if (res.status === 200 && res.data.result) {
+		return scrapeLatest(res.data.result, site, true, options);
+	}
+
+	return res.status;
+}
+
 function scrapeRelease(data, url, channel, networkName, options) {
 	if (Array.isArray(data)) {
 		return null;
@@ -192,140 +300,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	return release;
 }
 
-function getUrl(site) {
-	const { searchParams, pathname } = new URL(site.url);
-
-	// if (search.match(/\?site=\d+/)) {
-	if (searchParams.has('site') || /\/site\/\d+/.test(pathname)) {
-		return site.url;
-	}
-
-	if (site.parameters?.native) {
-		return `${site.url}/scenes`;
-	}
-
-	if (site.parameters?.extract) {
-		return `${site.url}/scenes`;
-	}
-
-	if (site.parameters?.siteId) {
-		return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
-	}
-
-	throw new Error(`Aylo site '${site.name}' (${site.url}) not supported`);
-}
-
-async function getSession(site, parameters, url) {
-	// if (site.slug === 'aylo' || site.parameters?.parentSession === false) {
-	if (site.slug === 'aylo') {
-		// most MG sites have a parent network to acquire a session from, don't try to acquire session from mindgeek.com for independent channels
-		return null;
-	}
-
-	const cookieJar = new CookieJar();
-	const session = http.session({ cookieJar });
-
-	const sessionUrl = site.parameters?.siteId && !(site.parameters?.native || site.parameters?.childSession || site.parent?.parameters?.childSession || site.parameters?.parentSession === false)
-		? site.parent.url
-		: (url || site.url);
-
-	/*
-	await unprint.browserRequest(sessionUrl, {
-		browser: {
-			headless: false,
-		},
-		async control() {
-			await new Promise((resolve) => { setTimeout(() => resolve(), 10000); });
-		},
-	});
-	*/
-
-	const res = await http.get(sessionUrl, {
-		session,
-		headers: {
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-			Connection: 'keep-alive',
-			'User-Agent': 'HTTPie/3.2.1',
-		},
-		interval: parameters?.interval,
-		concurrency: parameters?.concurrency,
-		parse: false,
-	});
-
-	if (res.status === 200) {
-		const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
-		const { instance_token: instanceToken } = cookie.parse(cookieString);
-
-		if (instanceToken) {
-			return { session, instanceToken };
-		}
-	}
-
-	throw new Error(`Failed to acquire Aylo session (${res.statusCode})`);
-}
-
-async function fetchLatest(site, page = 1, options) {
-	const url = getUrl(site);
-	const { searchParams, pathname } = new URL(url);
-	const siteId = searchParams.get('site') || Number(pathname.match(/\/site\/(\d+)\//)?.[1]);
-
-	if (!siteId && !site.parameters?.native && !site.parameters?.extract) {
-		return null;
-	}
-
-	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
-		? options.beforeNetwork
-		: await getSession(site, options.parameters, url);
-
-	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
-	const limit = 24;
-	const apiUrl = site.parameters?.native || site.parameters?.extract
-		? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
-		: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
-
-	const res = await http.get(apiUrl, {
-		interval: options.parameters.interval,
-		concurrency: options.parameters.concurrency,
-		headers: {
-			Instance: instanceToken,
-			Origin: site.url,
-			Referer: url,
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-		},
-	});
-
-	if (res.status === 200 && res.body.result) {
-		return scrapeLatest(res.body.result, site, false, options);
-	}
-
-	return res.statusCode;
-}
-
-async function fetchUpcoming(site, page, options) {
-	const url = getUrl(site);
-	const { session, instanceToken } = await getSession(site, options.parameters);
-
-	const apiUrl = 'https://site-api.project1service.com/v2/upcoming-releases';
-
-	const res = await http.get(apiUrl, {
-		session,
-		interval: options.parameters.interval,
-		concurrency: options.parameters.concurrency,
-		headers: {
-			Instance: instanceToken,
-			Origin: site.url,
-			Referer: url,
-			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
-		},
-	});
-
-	if (res.statusCode === 200 && res.body.result) {
-		return scrapeLatest(res.body.result, site, true, options);
-	}
-
-	return res.statusCode;
-}
-
 async function fetchRelease(url, site, baseScene, options) {
 	if (baseScene?.entryId && !baseScene.shallow && !options.parameters.forceDeep) {
 		// overview and deep data is the same, don't hit server unnecessarily
@@ -333,10 +307,9 @@ async function fetchRelease(url, site, baseScene, options) {
 	}
 
 	const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
-	const { session, instanceToken } = options.beforeFetchScenes || await getSession(site, options.parameters);
+	const { instanceToken } = options.beforeFetchScenes || await getSession(site, options.parameters);
 
-	const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
+	const res = await unprint.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
-		session,
 		interval: options.parameters.interval,
 		concurrency: options.parameters.concurrency,
 		headers: {
@@ -345,16 +318,16 @@ async function fetchRelease(url, site, baseScene, options) {
 		},
 	});
 
-	if (res.status === 200 && res.body.result) {
+	if (res.status === 200 && res.data.result) {
 		return {
-			scene: scrapeRelease(res.body.result, url, site, null, options),
+			scene: scrapeRelease(res.data.result, url, site, null, options),
 		};
 	}
 
 	return null;
 }
 
-function scrapeProfile(data, networkName, _releases = []) {
+function scrapeProfile(data, _networkName, _releases = []) {
 	const profile = {
 		description: data.bio,
 		aliases: data.aliases.filter(Boolean),
@@ -368,7 +341,7 @@ function scrapeProfile(data, networkName, _releases = []) {
 		profile.measurements = data.measurements;
 	}
 
-	profile.dateOfBirth = qu.parseDate(data.birthday);
+	profile.dateOfBirth = unprint.extractDate(data.birthday);
 	profile.birthPlace = data.birthPlace;
 	profile.height = inchesToCm(data.height);
 	profile.weight = lbsToKg(data.weight);
@@ -407,10 +380,9 @@ function scrapeProfile(data, networkName, _releases = []) {
 
 async function fetchProfile({ name: actorName }, { entity, parameters }, include) {
 	// const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
-	const { session, instanceToken } = await getSession(entity, parameters);
+	const { instanceToken } = await getSession(entity, parameters);
 
-	const res = await http.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
+	const res = await unprint.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
-		session,
 		interval: parameters.interval,
 		concurrency: parameters.concurrency,
 		headers: {
@@ -419,14 +391,13 @@ async function fetchProfile({ name: actorName }, { entity, parameters }, include
 		},
 	});
 
-	if (res.statusCode === 200) {
+	if (res.status === 200) {
-		const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
+		const actorData = res.data.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
 
 		if (actorData) {
 			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
 
-			const actorReleasesRes = include.includeActorScenes && await http.get(actorReleasesUrl, {
+			const actorReleasesRes = include.includeActorScenes && await unprint.get(actorReleasesUrl, {
-				session,
 				interval: parameters.interval,
 				concurrency: parameters.concurrency,
 				headers: {
@@ -434,8 +405,8 @@ async function fetchProfile({ name: actorName }, { entity, parameters }, include
 				},
 			});
 
-			if (actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
+			if (actorReleasesRes.status === 200 && actorReleasesRes.data.result) {
-				return scrapeProfile(actorData, entity.slug, actorReleasesRes.body.result);
+				return scrapeProfile(actorData, entity.slug, actorReleasesRes.data.result);
 			}
 
 			return scrapeProfile(actorData, entity.slug, []);

src/scrapers/aziani.js

@@ -6,11 +6,13 @@ const { decode } = require('html-entities');
 const slugify = require('../utils/slugify');
 const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
 
+const tagKeys = ['tags', 'category', 'categories'];
+
 function scrapeScene(data, channel, parameters) {
 	const release = {};
 
 	release.entryId = data.cms_set_id;
-	release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
+	release.url = `${parameters.scene || channel.url}${parameters.scenePath || '/video'}/${parameters.entryIdSlug ? data.slug : data.cms_set_id}`;
 
 	release.title = data.name;
 	release.description = data.description;
@@ -20,7 +22,7 @@ function scrapeScene(data, channel, parameters) {
 
 	release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models' || dataType.data_type === 'Talent')?.data_values.map((actor) => ({
 		name: actor.name,
-		url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
+		url: `${channel.url}${parameters.modelPath || '/model'}/${parameters.entryIdSlug && actor.slug ? actor.slug : actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
 	}));
 
 	release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
@@ -29,7 +31,7 @@ function scrapeScene(data, channel, parameters) {
 	}));
 
 	release.tags = data.data_types
-		.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
+		.filter((dataType) => tagKeys.includes(dataType.data_type?.toLowerCase()))
 		.flatMap((tags) => tags.data_values.map((tag) => tag.name));
 
 	const poster = data.preview_formatted.thumb;
@@ -43,7 +45,7 @@ function scrapeScene(data, channel, parameters) {
 			.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
 	}
 
-	release.caps = data.content
+	release[parameters.imageType || 'caps'] = data.content
 		?.filter((item) => item.content_type === 'image' && item.content?.[0])
 		.map((item) => [
 			item.content[0],
@@ -66,13 +68,14 @@ function scrapeScene(data, channel, parameters) {
 		release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
 	}
 
-	release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
+	release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '') || null;
 
 	return release;
 }
 
 async function getBlockId(slug, dataSource, entity, parameters) {
-	const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
+	const res = await unprint.get(`${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -109,9 +112,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
 		data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
 	}).toString();
 
-	const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
+	const url = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
 
 	const res = await unprint.get(url, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: channel.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -126,20 +130,21 @@ async function fetchLatest(channel, page = 1, { parameters }) {
 }
 
 async function fetchScene(url, entity, _baseRelease, { parameters }) {
-	const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
+	const entryId = new URL(url).pathname.match(/\/videos?\/([\w-]+)/)[1];
 
 	if (!entryId) {
 		return null;
 	}
 
-	const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
+	const blockId = await getBlockId(`${parameters.videos || '/video'}/${parameters.entryIdSlug ? ':slug' : ':id'}`, entryId, entity, parameters);
 
 	if (!blockId) {
 		return null;
 	}
 
 	const query = new URLSearchParams({
-		cms_set_ids: entryId,
+		cms_set_ids: parameters.entryIdSlug ? '' : entryId,
+		...parameters.entryIdSlug ? { slug: entryId } : null,
 		cms_area_id: parameters.areaId,
 		cms_block_id: blockId,
 		content: 1,
@@ -150,9 +155,10 @@ async function fetchScene(url, entity, _baseRelease, { parameters }) {
 		content_count: 1,
 	}).toString();
 
-	const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
+	const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
 
 	const res = await unprint.get(apiUrl, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': parameters.areaId,
@@ -174,14 +180,18 @@ function scrapeProfile(data, entity, parameters) {
 		decode(detail.value || detail.content_formatted || detail.content),
 	]));
 
-	profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
+	profile.url = `${entity.url}/model/${parameters.entryIdSlug ? data.slug : data.cms_data_value_id}`;
 	profile.entryId = data.cms_data_value_id;
 
 	profile.description = data.description;
 
 	profile.gender = bio.gender?.toLowerCase();
 	profile.age = bio.age;
-	profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
+	profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ })
+		|| unprint.extractDate(bio.date_of_birth, 'yyyy-MM-dd');
+
+	profile.birthPlace = bio.birthplace;
+	profile.ethnicity = bio.ethnicity;
 
 	profile.measurements = bio.measurements;
 	profile.height = feetInchesToCm(bio.height);
@@ -190,7 +200,9 @@ function scrapeProfile(data, entity, parameters) {
 	profile.hairColor = bio.hair_color;
 	profile.eyeColor = bio.eye_color;
 
-	const avatar = bio.thumbnail?.image[0];
+	const avatar = bio.thumbnail?.image?.[0] // aziani
+		|| bio.biopic?.image?.[0] // cosplayground profile
+		|| bio.headshot?.image?.[0]; // cosplayground headshot
 
 	if (avatar) {
 		profile.avatar = {
@@ -201,36 +213,41 @@ function scrapeProfile(data, entity, parameters) {
 		};
 	}
 
+	profile.socials = [bio.onlyfans, bio.instagram].filter(Boolean);
+
 	return profile;
 }
 
 async function fetchProfile({ name, url }, { entity, parameters }) {
-	if (!url) {
+	if (!url && !parameters.entryIdSlug) {
-		// no easy search option
+		// no easy search option and numeric ID is non-derivable
 		return null;
 	}
 
-	const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
+	const actorId = url && new URL(url).pathname.match(/models?\/(\d+)/)?.[1];
 
-	if (!actorId) {
+	if (!actorId && !parameters.entryIdSlug) {
 		return null;
 	}
 
-	const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
+	const slug = slugify(name);
+	const blockId = await getBlockId(`${parameters.modelPath || '/model'}/${parameters.entryIdSlug ? ':slug' : 'id'}`, { models: name, id: actorId, slug }, entity, parameters);
 
 	if (!blockId) {
 		return null;
 	}
 
 	const query = new URLSearchParams({
-		cms_data_value_ids: actorId,
+		cms_data_value_ids: actorId || '',
+		...parameters.entryIdSlug ? { slug } : null,
 		cms_block_id: blockId,
 		cms_data_type_id: 4,
 	}).toString();
 
-	const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
+	const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/data-values?${query}`;
 
 	const res = await unprint.get(apiUrl, {
+		interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
 		headers: {
 			Referer: entity.url,
 			'x-nats-cms-area-id': entity.parameters.areaId,

src/scrapers/jaxslayher.js

@@ -0,0 +1,257 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const slugify = require('../utils/slugify');
+const { convert } = require('../utils/convert');
+const tryUrls = require('../utils/try-urls');
+
+// parse Next/React data tree
+async function parsePayload(rawText) {
+	const { createFlightResponse, processBinaryChunk } = await import('@rsc-parser/react-client'); // eslint-disable-line import/no-unresolved
+
+	const response = createFlightResponse(false);
+	const bytes = new TextEncoder().encode(rawText);
+
+	processBinaryChunk(response, bytes);
+
+	return response._chunks;
+}
+
+function traverseData(node, predicate, resultDepth = 3, seen = new WeakSet(), ancestors = []) {
+	if (node === null || typeof node !== 'object') {
+		return null;
+	}
+
+	if (seen.has(node)) {
+		return null;
+	}
+
+	seen.add(node);
+
+	if (predicate(node)) {
+		if (resultDepth === 0) {
+			return node;
+		}
+
+		const index = ancestors.length - resultDepth;
+
+		return ancestors[Math.max(index, 0)] ?? node;
+	}
+
+	const values = Array.isArray(node) ? node : Object.values(node);
+	const nextAncestors = [...ancestors, node];
+
+	let result = null;
+
+	values.some((value) => {
+		result = traverseData(value, predicate, resultDepth, seen, nextAncestors);
+		return result !== null;
+	});
+
+	return result;
+}
+
+function getPoster(videoId, channel) {
+	if (!videoId) {
+		return null;
+	}
+
+	return [
+		`${channel.origin}/images/updates/${videoId}-1280.jpg`,
+		`${channel.origin}/images/updates/${videoId}-720.jpg`,
+		`${channel.origin}/images/updates/${videoId}.jpg`, // 502px
+		`${channel.origin}/images/updates/${videoId}-360.jpg`,
+	];
+}
+
+function getVideo(videoId, path = '/trailers', channel) {
+	if (!videoId) {
+		return null;
+	}
+
+	return [
+		`${channel.origin}${path}/${videoId}.mp4`,
+		`${channel.origin}${path}/${videoId}-med.mp4`,
+	];
+}
+
+function scrapeAll(scenes, channel) {
+	return scenes.map((data) => {
+		const release = {};
+
+		release.url = `${channel.origin}/tour/video/${data.slug}`;
+		release.entryId = data.id; // URL slugs are not unique, some pages on original site unreachable!
+
+		release.attributes = {
+			dataEntryId: data.id, // not used in URL, store as secondary
+			slug: data.slug,
+		};
+
+		release.title = data.title;
+		release.date = new Date(data.release_date);
+
+		release.actors = data.models.map((model) => ({
+			name: model.name,
+			url: `${channel.origin}/tour/model/${slugify(model.slug)}`, // for some reason they are underscored, while URL expects dashes
+		}));
+
+		release.tags = data.tags.map((tag) => tag.tag_name);
+
+		release.poster = getPoster(data.id, channel);
+
+		release.teaser = getVideo(data.id, '/video_thumbs', channel);
+		release.trailer = getVideo(data.id, '/trailers', channel);
+
+		return release;
+	});
+}
+
+async function fetchLatest(channel, page = 1) {
+	// HTML does not contain tags, hence we jump through hoops to fetch the data directly
+	const res = await unprint.get(`${channel.origin}/tour/videos/most-recent/page/${page}.txt`, {
+		headers: {
+			rsc: 1,
+		},
+	});
+
+	if (res.ok) {
+		const payload = await parsePayload(res.body);
+		const sceneData = traverseData(payload, (item) => item.content_type === 'video');
+
+		if (sceneData) {
+			return scrapeAll(sceneData.map((item) => item.props.item), channel);
+		}
+	}
+
+	return res.status;
+}
+
+function scrapeScene(payload, { url, channel, baseRelease }) {
+	const release = {};
+
+	const sceneData = traverseData(payload, (item) => Object.hasOwn(item, 'videoID'), 2);
+	const durationData = traverseData(sceneData, (item) => /\d{2}:\d{2}:\d{2}/.test(item.children), 0);
+	const videoData = traverseData(sceneData, (item) => Object.hasOwn(item, 'videoID'), 0);
+	const dateData = traverseData(sceneData, (item) => item.children?.includes?.('2026'), 0);
+
+	const videoId = videoData?.videoID;
+
+	release.duration = unprint.extractDuration(durationData?.children);
+
+	if (baseRelease?.entryId) {
+		// base release data is much more concise, duration only thing missing
+		return release;
+	}
+
+	release.entryId = videoId;
+
+	release.attributes = {
+		dataEntryId: videoId,
+		slug: new URL(url).pathname.match(/\/video\/([\w-]+)/)?.[1],
+	};
+
+	release.title = videoData?.title;
+	release.date = unprint.extractDate(dateData.children, 'MMM DD/YYYY', { match: null });
+
+	release.poster = getPoster(videoId, channel);
+	release.teaser = getVideo(videoId, '/video_thumbs', channel);
+	release.trailer = getVideo(videoId, '/trailers', channel);
+
+	return release;
+}
+
+async function fetchScene(url, channel, baseRelease) {
+	const dataUrl = `${url.replace(/\/*$/, '')}.txt`;
+
+	const res = await unprint.get(dataUrl, {
+		headers: {
+			rsc: 1,
+		},
+	});
+
+	if (res.ok) {
+		const payload = await parsePayload(res.body);
+
+		return scrapeScene(payload, { url, channel, baseRelease });
+	}
+
+	return res.status;
+}
+
+function getBioItem(bioData, key) {
+	if (!bioData) {
+		return null;
+	}
+
+	const data = traverseData(bioData, (item) => typeof item.children === 'string' && item.children.toLowerCase() === `${key}:`, 2);
+	const values = data[1]?.props.children;
+
+	const value = Array.isArray(values)
+		? values.find((itemValue) => (typeof itemValue === 'string' ? itemValue.trim() : itemValue))
+		: values;
+
+	if (value && String(value).toLowerCase() !== 'n/a') {
+		return value;
+	}
+
+	return null;
+}
+
+function scrapeProfile(payload, actorName, channel, url) {
+	const profile = { url };
+
+	const actorData = traverseData(payload, (item) => slugify(item.name) === slugify(actorName), 0);
+	const bioData = traverseData(payload, (item) => item.className === 'description_main', 0);
+	const descriptionData = traverseData(payload, (item) => item.className === 'description_top', 0);
+	const description = traverseData(descriptionData, (item) => item.className === 'info', 0)?.children;
+
+	profile.entryId = actorData?.id;
+
+	profile.birthPlace = getBioItem(bioData, 'born');
+
+	profile.dateOfBirth = getBioItem(bioData, 'birthdate');
+	profile.age = getBioItem(bioData, 'age');
+
+	profile.eyes = getBioItem(bioData, 'eyes');
+	profile.hairColor = getBioItem(bioData, 'hair');
+	profile.weight = convert(getBioItem(bioData, 'weight'), 'lb', 'kg');
+	profile.height = convert(getBioItem(bioData, 'height'), 'cm');
+	profile.measurements = getBioItem(bioData, 'measurements');
+
+	if (!/no bio available/i.test(description)) {
+		profile.description = description;
+	}
+
+	if (actorData.is_thumbed && actorData.id) {
+		profile.avatar = [
+			`${channel.origin}/images/models/${actorData.id}.jpg`, // 628px
+			`${channel.origin}/images/models/${actorData.id}-516.jpg`,
+			`${channel.origin}/images/models/${actorData.id}-408.jpg`,
+			`${channel.origin}/images/models/${actorData.id}-300.jpg`,
+		];
+	}
+
+	return profile;
+}
+
+async function fetchProfile({ name: actorName, url: actorUrl }, entity) {
+	const { res, url } = await tryUrls([
+		actorUrl && `${actorUrl}.txt`,
+		`${entity.url}/tour/model/${slugify(actorName, '-')}.txt`,
+	]);
+
+	if (res.ok) {
+		const payload = await parsePayload(res.body);
+
+		return scrapeProfile(payload, actorName, entity, url.replace(/\.txt$/, ''));
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchProfile,
+	fetchScene,
+};

src/scrapers/julesjordan.js

@@ -1,44 +1,38 @@
 'use strict';
 
 const util = require('util');
-const Promise = require('bluebird');
 const unprint = require('unprint');
 
 const argv = require('../argv');
 const { heightToCm } = require('../utils/convert');
 const slugify = require('../utils/slugify');
+const tryUrls = require('../utils/try-urls');
 
-function getEntryId(html) {
+function getEntryId(url) {
-	const entryId = html.match(/showtagform\((\d+)\)/);
+	if (!url) {
-
+		return null;
-	if (entryId) {
-		return entryId[1];
 	}
 
-	const setIdIndex = html.indexOf('setid:"');
+	const entryId = new URL(url).pathname.split('/').at(-1).match('(.*?)_vids.html');
 
-	if (setIdIndex) {
+	if (entryId) {
-		return html.slice(setIdIndex, html.indexOf(',', setIdIndex)).match(/\d+/)?.[0];
+		return slugify(entryId[1]);
 	}
 
 	return null;
 }
 
-function getEntryIdFromTitle(release) {
+function scrapeAll(scenes, site) {
-	// return slugify([release.title, release.date && unprint.formatDate(release.date, 'YYYY-MM-DD')]); // date not shown on updates page
+	return scenes.map(({ query }) => {
-	// return slugify(release.title);
-	return slugify([release.title, ...(release.actors?.map((actor) => actor.name || actor).toSorted() || [])]);
-}
-
-function scrapeAll(scenes, site, entryIdFromTitle) {
-	return scenes.map(({ element, query }) => {
 		const release = {};
-		const title = query.content('.content_img div, .dvd_info > a, a.update_title, .update_title a, a[title] + a[title], .overlay-text')
+		const title = query.content('.jj-card-title, .content_img div, .dvd_info > a, a.update_title, .update_title a, a[title] + a[title], .overlay-text')
 			|| query.content('a[title*=" "]');
 
 		release.title = title?.slice(0, title.match(/starring:/i)?.index || Infinity).trim();
-		release.url = query.url('.content_img a, .dvd_info > a, a.update_title, .update_title a, a[title]');
+		release.url = query.url('.jj-card-thumb, .content_img a, .dvd_info > a, a.update_title, .update_title a, a[title]');
-		release.date = query.date('.update_date', ['MM/DD/YYYY', 'YYYY-MM-DD']);
+		release.date = query.date('.jj-card-date, .update_date', ['MM/DD/YYYY', 'YYYY-MM-DD', 'MMMM D, YYYY']);
+
+		release.entryId = getEntryId(release.url);
 
 		release.actors = query.all('.content_img .update_models a, .update_models a').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
@@ -70,17 +64,16 @@ function scrapeAll(scenes, site, entryIdFromTitle) {
 				return null;
 			}).filter(Boolean);
 
-		const teaserScript = query.html('script');
+		release.teaser = query.video('.jj-card-video', { attribute: 'data-src' });
 
-		if (teaserScript) {
+		if (!release.teaser) {
-			release.teaser = teaserScript.slice(teaserScript.indexOf('http'), teaserScript.indexOf('.mp4') + 4);
+			const teaserScript = query.html('script');
+
+			if (teaserScript) {
+				release.teaser = teaserScript.slice(teaserScript.indexOf('http'), teaserScript.indexOf('.mp4') + 4);
+			}
 		}
 
-		release.entryId = (entryIdFromTitle && getEntryIdFromTitle(release))
-			|| element.dataset.setid
-			|| query.element('.rating_box')?.dataset.id
-			|| query.attribute('a img', 'id')?.match(/set-target-(\d+)/)?.[1];
-
 		return release;
 	});
 }
@@ -91,7 +84,7 @@ async function fetchLatest(site, page = 1, include, preData, entryIdFromTitle =
 		: `${site.url}/trial/categories/movies_${page}_d.html`;
 
 	// const res = await http.get(url);
-	const res = await unprint.get(url, { selectAll: '.update_details, .grid-item' });
+	const res = await unprint.get(url, { selectAll: '.scenes-listing-grid .jj-content-card' });
 
 	if (res.ok) {
 		return scrapeAll(res.context, site, typeof site.parameters?.entryIdFromTitle === 'boolean' ? site.parameters.entryIdFromTitle : entryIdFromTitle);
@@ -100,7 +93,8 @@ async function fetchLatest(site, page = 1, include, preData, entryIdFromTitle =
 	return res.status;
 }
 
-function scrapeUpcoming(scenes, channel) {
+/* disable until we have entryId solution
+function scrapeUpcoming(scenes, _channel) {
 	return scenes.map(({ query, html }) => {
 		const release = {};
 
@@ -135,6 +129,7 @@ async function fetchUpcoming(site) {
 
 	return res.status;
 }
+*/
 
 function extractLegacyTrailer(html, context) {
 	const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
@@ -194,17 +189,19 @@ function getPhotos(query, release, context) {
 async function scrapeScene({ html, query }, context) {
 	const release = {};
 
-	release.title = query.content('.title_bar_hilite, .movie_title');
+	release.entryId = getEntryId(context.url);
-	release.description = query.content('.update_description') || query.text('//div[./span[contains(text(), "Description")]]');
 
-	release.date = query.date(['.update_date', '//div[./span[contains(text(), "Date")]]'], ['MM/DD/YYYY', 'YYYY-MM-DD']);
+	release.title = query.content('.scene-title, .title_bar_hilite, .movie_title');
+	release.description = query.content('.scene-desc, .update_description') || query.text('//div[./span[contains(text(), "Description")]]');
 
-	release.actors = query.all('.backgroundcolor_info > .update_models a, .item .update_models a, .player-scene-description .update_models a').map((actorEl) => ({
+	release.date = query.date(['.meta-item:nth-child(2) .val, .update_date', '//div[./span[contains(text(), "Date")]]'], ['MM/DD/YYYY', 'YYYY-MM-DD', 'MMMM D, YYYY']);
+
+	release.actors = query.all('.meta-item .update_models a, .backgroundcolor_info > .update_models a, .item .update_models a, .player-scene-description .update_models a').map((actorEl) => ({
 		name: unprint.query.content(actorEl),
 		url: unprint.query.url(actorEl, null),
 	}));
 
-	release.tags = query.contents('.update_tags a, .player-scene-description a[href*="/categories"]');
+	release.tags = query.contents('.scene-cats a, .update_tags a, .player-scene-description a[href*="/categories"]');
 	release.director = release.tags?.find((tag) => ['mike john', 'van styles'].includes(tag?.trim().toLowerCase()));
 
 	const posterPath = query.poster('#video-player', { forceGetAttribute: true }) // without getAttribute, missing poster is returned as page URL
@@ -245,7 +242,7 @@ async function scrapeScene({ html, query }, context) {
 		// #images img selects a list of images that is present on every page; the JJ website removes the ones that failed to load with JS (lol)
 		release.photos = [
 			...context.baseRelease?.photos?.map((sources) => sources.at(-1).src) || [],
-			...query.imgs('#images img'),
+			...query.imgs('.tp-photos-strip img, #images img'),
 			...query.imgs('img.update_thumb', { attribute: 'src0_1x' }),
 		].filter(Boolean).map((source) => Array.from(new Set([
 			source.replace(/.jpg$/, '-full.jpg'),
@@ -259,20 +256,20 @@ async function scrapeScene({ html, query }, context) {
 		})));
 	}
 
-	if (query.exists('.player-scene-description a[href*="/dvd"]')) {
+	const movieQuery = '//div[@class="meta-item"][div[@class="lbl" and text()="Movie"]]/div[@class="val"]/a';
+
+	if (query.exists(movieQuery)) {
 		release.movie = {
-			url: query.url('.player-scene-description a[href*="/dvd"]'),
+			url: query.url(movieQuery),
-			title: query.content('.player-scene-description a[href*="/dvd"]'),
+			title: query.content(movieQuery),
 		};
 
-		release.movie.entryId = new URL(release.movie.url).pathname.split('/').slice(-1)[0]?.replace('.html', '').toLowerCase();
+		release.movie.entryId = new URL(release.movie.url).pathname.match(/\/dvds\/(.*?).html/)?.[1];
 	}
 
 	release.stars = query.number('.avg_rating');
 
-	release.entryId = context.entity.parameters?.entryIdFromTitle
+	release.qualities = query.contents('.res-item .res-lbl').map((resolution) => Number(resolution.match(/\d+$/)?.[0])).filter(Boolean);
-		? getEntryIdFromTitle(release)
-		: getEntryId(html);
 
 	return release;
 }
@@ -280,32 +277,35 @@ async function scrapeScene({ html, query }, context) {
 function scrapeMovie({ query }, { url }) {
 	const movie = {};
 
-	movie.entryId = new URL(url).pathname.split('/').slice(-1)[0]?.replace('.html', '').toLowerCase();
+	movie.entryId = new URL(url).pathname.match(/\/dvds\/(.*?).html/)?.[1];
-	movie.title = query.attribute('meta[property="og:title"]', 'content');
+	movie.title = query.attribute('meta[property="og:title"]', 'content')?.replace(/\s*-\s*jules jordan/i, '');
 
 	movie.covers = [query.img('img.dvd_box')]; // -2x etc is likely upscaled
 
-	const sceneTitles = query.contents('.title-heading-content-black-dvd');
+	const descriptionDate = query.content('meta[name="description"]', { attribute: 'content' })?.match(/released (\w+ \d{4})/i)?.[1];
 
-	const scenes = query.all('.grid-container-scene').map((sceneEl, index) => {
+	if (descriptionDate) {
+		movie.date = unprint.extractDate(descriptionDate, 'MMMM YYYY', { match: null });
+		movie.datePrecision = 'month';
+	}
+
+	const scenes = query.all('.dvd-scene').map((sceneEl) => {
 		const scene = {};
 
-		scene.url = unprint.query.url(sceneEl, 'a[href*="/scenes"]');
+		scene.url = unprint.query.url(sceneEl, '.dvd-watch-btn');
-		scene.title = sceneTitles[index];
+		scene.entryId = getEntryId(scene.url);
 
-		scene.date = unprint.query.date(sceneEl, '//span[contains(@class, "dvd-scene-description") and span[contains(text(), "Date")]]', 'MM/DD/YYYY');
+		scene.title = unprint.query.content(sceneEl, '.dvd-hero-title');
-		scene.actors = unprint.query.contents(sceneEl, '.update_models a');
+		scene.date = unprint.query.date(sceneEl, '.dvd-hero-date', 'MM/DD/YYYY');
+		scene.actors = unprint.query.contents(sceneEl, '.dvd-hero .update_models a');
 
-		scene.entryId = getEntryIdFromTitle(scene);
+		scene.photos = unprint.query.imgs(sceneEl, '.dvd-img-strip img');
 
 		return scene;
 	});
 
 	movie.scenes = scenes?.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
 
-	movie.date = movie.scenes?.[0]?.date;
-	movie.datePrecision = 'month';
-
 	return movie;
 }
 
@@ -339,48 +339,34 @@ function scrapeProfile({ query }, url, entity) {
 		verifyType: 'image',
 	}));
 
-	profile.scenes = scrapeAll(unprint.initAll(query.all('.grid-item')), entity, true);
+	profile.scenes = scrapeAll(unprint.initAll(query.all('.mbp-scenes-grid .jj-content-card, .grid-item')), entity, true);
 
 	return profile;
 }
 
-async function fetchProfile({ name: actorName, url }, entity) {
+async function fetchProfile({ name: actorName, url: actorUrl }, entity) {
 	const actorSlugA = slugify(actorName, '');
 	const actorSlugB = slugify(actorName, '-');
 
 	const urls = [
-		url,
+		actorUrl,
 		`${entity.parameters?.profile || `${entity.url}/trial/models`}/${actorSlugA}.html`,
 		`${entity.parameters?.profile || `${entity.url}/trial/models`}/${actorSlugB}.html`,
 	];
 
-	return urls.reduce(async (chain, profileUrl) => {
+	const { res, url } = await tryUrls(urls);
-		const profile = await chain;
 
-		if (profile) {
+	if (res.ok) {
-			return profile;
+		return scrapeProfile(res.context, url, entity);
-		}
+	}
 
-		if (!profileUrl) {
+	return res.status;
-			return null;
-		}
-
-		const res = await unprint.get(profileUrl, {
-			followRedirects: false,
-		});
-
-		if (res.ok) {
-			return scrapeProfile(res.context, profileUrl, entity);
-		}
-
-		return null;
-	}, Promise.resolve());
 }
 
 module.exports = {
 	fetchLatest,
 	fetchProfile,
-	fetchUpcoming,
+	// fetchUpcoming,
 	scrapeScene,
 	scrapeMovie,
 };

src/scrapers/kellymadison.js

@@ -5,7 +5,6 @@ const unprint = require('unprint');
 // const { parse } = require('csv-parse/sync');
 
 const slugify = require('../utils/slugify');
-const http = require('../utils/http');
 const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
 
 const thumbKeyRegex = /(thumb\d+_url)|(episode_thumb_image_\d+_url)/;
@@ -84,14 +83,14 @@ function scrapeSceneApi(data, channel, parameters) {
 
 async function fetchLatestApi(channel, page = 1, { parameters }) {
 	// JSON API doesn't return poster images, CSV API doesn't have pagination. UPDATE: requested and received both, yet to test
-	const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
+	const res = await unprint.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
 		headers: {
 			Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`,
 		},
 	});
 
 	if (res.ok) {
-		return res.body.data.map((data) => scrapeSceneApi(data, channel, parameters));
+		return res.data.data.map((data) => scrapeSceneApi(data, channel, parameters));
 	}
 
 	return res.status;

src/scrapers/kink.js

@@ -18,7 +18,7 @@ function scrapeAll(scenes, entity) {
 		release.entryId = release.shootId;
 
 		release.title = query.content('.card-body a[href*="/shoot"]').trim();
-		release.date = query.date('small > span', 'MMM D, YYYY');
+		release.date = query.date('.small > span', 'MMM D, YYYY');
 
 		release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
@@ -54,7 +54,11 @@ function scrapeAll(scenes, entity) {
 			// no photos
 		}
 
-		release.trailer = `https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`;
+		release.trailer = [
+			query.video('.ratio-thumbnail img', { attribute: 'data-trailer-url' }),
+			`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
+			`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
+		];
 
 		release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
 		release.rating = query.number('.thumb-up') / 10;
@@ -64,10 +68,13 @@ function scrapeAll(scenes, entity) {
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `${channel.parent.url}/search?type=shoots&channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
+	const url = `${channel.parent.url}/shoots?channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
 
 	const res = await unprint.browser(url, {
 		selectAll: '.container .card',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -129,6 +136,7 @@ function scrapeScene({ query }, url, entity) {
 			src: source.url,
 			quality: source.resolution,
 		})) || []),
+		`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
 		`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
 	];
 
@@ -143,7 +151,11 @@ function scrapeScene({ query }, url, entity) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await unprint.browser(url);
+	const res = await unprint.browser(url, {
+		cookies: {
+			age_gate_accepted: '1',
+		},
+	});
 
 	if (res.status === 200) {
 		const scene = scrapeScene(res.context, url, channel);
@@ -197,6 +209,9 @@ async function getActorUrl({ name: actorName, url }, networkUrl) {
 	// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
 	const res = await unprint.get(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`, {
 		interface: 'request',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -217,7 +232,11 @@ async function fetchProfile(actor, entity) {
 	const actorUrl = await getActorUrl(actor, networkUrl);
 
 	if (actorUrl) {
-		const actorRes = await unprint.browser(actorUrl);
+		const actorRes = await unprint.browser(actorUrl, {
+			cookies: {
+				age_gate_accepted: '1',
+			},
+		});
 
 		if (actorRes.status === 200) {
 			return scrapeProfile(actorRes.context, actorUrl);

src/scrapers/manyvids.js

@@ -0,0 +1,241 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const slugify = require('../utils/slugify');
+
+function splitTag(tag) {
+	return tag
+		.replace(/([a-z])([A-Z])/g, '$1 $2')
+		.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') // handle sequence of caps
+		.toLowerCase()
+		.trim();
+}
+
+async function getTrailer(videoId) {
+	if (!videoId) {
+		return null;
+	}
+
+	const res = await unprint.get(`https://api.manyvids.com/store/video/${videoId}/private`);
+
+	if (res.ok && res.data?.statusCode === 200) {
+		return res.data.data.teaser.filepath;
+	}
+
+	return null;
+}
+
+/*
+function getModelUrl(modelData) {
+	const slug = modelData.slug || modelData.profileUrl?.match(/\/profile\/\d+\/(\w+)/i)?.[1] || null;
+
+	if (slug) {
+		return `https://${slug}.manyvids.com`;
+	}
+
+	return null;
+}
+*/
+
+async function scrapeScene(data, context) {
+	const release = {};
+
+	release.url = data.url
+		? `https://www.manyvids.com${data.url}`
+		: `https://www.manyvids.com/Video/${data.id}/${data.slug || ''}`;
+
+	release.entryId = data.id;
+
+	release.attributes = {
+		price: data.price?.regular,
+		discount: data.price?.discountedPrice,
+		isCustomOrder: data.isCustomOrder,
+		isPremium: data.isPremium,
+		isPrivate: data.isPrivate,
+		isExclusive: data.isExclusive,
+		isStreamOnly: data.isStreamOnly,
+	};
+
+	release.title = data.title;
+	release.description = data.description;
+
+	release.date = new Date(data.launchDate);
+	release.duration = unprint.extractDuration(data.duration || data.videoDuration);
+
+	/* many channels operate more like a studio than a performer, so the channel information doesn't represent a person
+	const model = data.creator || data.model;
+
+	release.actors = [{
+		name: model.stageName || model.displayName,
+		entryId: model.id,
+		avatar: model.avatar?.url || model.avatar,
+		url: getModelUrl(model),
+	}];
+	*/
+
+	release.tags = [
+		data.is3D && '3d',
+		data.isAiGenerated && 'ai',
+		...data.tagList?.map((tag) => splitTag(tag.label)) || [],
+	].filter(Boolean);
+
+	release.poster = data.thumbnail?.url || data.screenshot || data.thumbnail; // thumbnail URL is full size, thumbnail direct is thumb sized
+
+	if (data.preview?.url) {
+		release.trailer = data.preview?.url;
+	} else if (context.includeTrailers) {
+		release.trailer = await getTrailer(data.id);
+	}
+
+	release.qualities = [data.height].filter(Boolean);
+
+	return release;
+}
+
+const profileIdRegex = /\/profile\/(\d+)/i;
+
+async function getProfileId(channel) {
+	if (channel.url) {
+		const { hostname, pathname } = new URL(channel.url);
+
+		if (profileIdRegex.test(pathname)) { // full profile URL
+			const urlProfileId = new URL(channel.url).pathname.match(profileIdRegex)?.[1];
+
+			if (urlProfileId) {
+				return urlProfileId;
+			}
+		}
+
+		if (/[\w-]+.manyvids.com/.test(hostname)) { // short profile URL
+			const idRes = await unprint.request(channel.url, {
+				method: 'head',
+				followRedirects: 0,
+			});
+
+			const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
+
+			if (profileId) {
+				return profileId;
+			}
+		}
+
+		// sadly we can't reliably reconstruct short profile URL from the slug
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const profileId = await getProfileId(channel);
+
+	if (profileId) {
+		const res = await unprint.get(`https://api.manyvids.com/store/videos/${profileId}?sort=newest&limit=100&page=${page}`);
+
+		if (res.ok && res.data?.statusCode === 200) {
+			return Promise.all(res.data.data.map(async (data) => scrapeScene(data, channel)));
+		}
+	}
+
+	return null;
+}
+
+async function fetchScene(url, channel, baseRelease, context) {
+	const videoId = new URL(url).pathname.match(/\/video\/(\d+)/i)?.[1];
+
+	if (!videoId) {
+		return null;
+	}
+
+	const res = await unprint.get(`https://api.manyvids.com/store/video/${videoId}`);
+
+	if (res.ok && res.data?.statusCode === 200) {
+		return scrapeScene(res.data.data, {
+			channel,
+			includeTrailers: context.includeTrailers,
+			isDeep: !!baseRelease.entryId,
+		});
+	}
+
+	return res.status;
+}
+
+function scrapeProfile(data) {
+	const profile = {};
+
+	profile.description = [data.bio, data.description].filter(Boolean).join(' | ') || null;
+
+	profile.placeOfResidence = data.location || null;
+	profile.ethnicity = data.ethnicity || null;
+	profile.hairColor = data.hairColor || null;
+
+	profile.gender = data.identification?.toLowerCase() || null;
+	profile.orientation = data.orientation;
+
+	profile.avatar = data.avatar || null;
+	profile.photos = [data.portrait].filter(Boolean);
+
+	profile.socials = [data.socLinkTwitter, data.socLinkInstagram, data.socLinkReddit].filter(Boolean);
+
+	const dateOfBirth = data.dob && unprint.extractDate(data.dob);
+
+	if (dateOfBirth && new Date().getFullYear() - dateOfBirth.getFullYear() > 18) {
+		// date of births that would not be legal are likely bogus
+		profile.dateOfBirth = dateOfBirth;
+	}
+
+	return profile;
+}
+
+async function getActorUrl(actor) {
+	if (actor.url) {
+		const idRes = await unprint.request(actor.url, {
+			method: 'head',
+			followRedirects: 0,
+		});
+
+		const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
+
+		if (profileId) {
+			return `https://api.manyvids.com/profile/profile/${profileId}`;
+		}
+	}
+
+	const res = await unprint.get(`https://api.manyvids.com/search/autocomplete?keywords=${slugify(actor.name, '+')}`);
+
+	if (res.ok) {
+		const model = res.data?.models?.find((modelResult) => modelResult.stage_name === actor.name || modelResult.username === actor.name);
+
+		if (model) {
+			const profileId = model.url?.match(/\/profile\/(\d+)/i)?.[1];
+
+			if (profileId) {
+				return `https://api.manyvids.com/profile/profile/${profileId}`;
+			}
+		}
+	}
+
+	return null;
+}
+
+async function fetchProfile(actor, entity) {
+	const actorUrl = await getActorUrl(actor);
+
+	if (actorUrl) {
+		const res = await unprint.get(actorUrl);
+
+		if (res.ok && res.data?.username) {
+			return scrapeProfile(res.data, entity);
+		}
+
+		return res.status;
+	}
+
+	return actorUrl;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchProfile,
+	fetchScene,
+};

src/scrapers/nubiles.js

@@ -21,14 +21,16 @@ function stripQuery(link) {
 }
 
 async function getPhotos(albumUrl) {
-	const res = await unprint.get(albumUrl, {
+	// aggressive Turnstile protection calls for browser
+	const res = await unprint.browser(albumUrl, {
 		selectAll: '.photo-thumb',
-		interface: 'request',
 	});
 
-	return res.ok
+	if (res.ok) {
-		? res.context.map(({ query }) => unprint.prefixUrl(query.element('source').srcset))
+		return res.context.map(({ query }) => query.sourceSet('source, picture img')).filter(Boolean);
-		: [];
+	}
+
+	return [];
 }
 
 function scrapeAll(scenes, entity) {
@@ -37,7 +39,7 @@ function scrapeAll(scenes, entity) {
 
 		release.title = query.content('.title a');
 		release.url = stripQuery(unprint.prefixUrl(query.url('.title a'), entity.url));
-		release.entryId = Number(new URL(release.url).pathname.match(/\/watch\/(\d+)/)[1]);
+		release.entryId = Number(new URL(release.url).pathname.match(/\/(?:watch|stream)\/(\d+)/)[1]);
 
 		release.date = query.date('.date', 'MMM D, YYYY');
 
@@ -51,7 +53,8 @@ function scrapeAll(scenes, entity) {
 			release.actors = query.content('.models', { trim: false })?.trim().split(/\s{2,}/);
 		}
 
-		const poster = query.sourceSet('img', 'data-srcset')?.[0];
+		const poster = query.sourceSet('img', 'data-srcset')?.[0]
+			|| query.sourceSet('img', 'data-src')?.[0];
 
 		release.poster = poster && [
 			poster.replace('_640', '_1280'),

src/scrapers/pornbox.js

@@ -33,7 +33,7 @@ function extractShootId(title) {
 	return title.trim().match(/[A-Z]{2,3}\d{3,4}\w?/)?.[0].toUpperCase();
 }
 
-async function scrapeScene(data, channel, include) {
+async function scrapeScene(data, channel, context) {
 	const release = {};
 	const entityUrl = new URL(channel.url).origin;
 
@@ -41,7 +41,10 @@ async function scrapeScene(data, channel, include) {
 
 	release.title = data.scene_name || data.custom_name;
 
-	release.url = `${entityUrl}/watch/${data.id}/${slugify(release.title, '_') || ''}`;
+	release.url = context.parameters?.usePornboxUrl
+		? `https://pornbox.com/application/watch-page/${data.id}`
+		: `${entityUrl}/watch/${data.id}/${slugify(release.title, '_') || ''}`;
+
 	release.shootId = extractShootId(release.title);
 
 	release.date = new Date(data.release_date || data.publish_date);
@@ -77,7 +80,7 @@ async function scrapeScene(data, channel, include) {
 
 	release.teaser = data.video_preview;
 
-	if (include?.includeTrailers) {
+	if (context?.includeTrailers) {
 		release.trailer = await getTrailer(data);
 		release.qualities = release.trailer?.map((src) => src.quality);
 	}
@@ -91,11 +94,11 @@ async function scrapeScene(data, channel, include) {
 	return release;
 }
 
-function scrapeAll(scenes, channel) {
+function scrapeAll(scenes, channel, context) {
-	return Promise.all(scenes.map(async (data) => scrapeScene(data, channel)));
+	return Promise.all(scenes.map(async (data) => scrapeScene(data, channel, context)));
 }
 
-async function fetchLatest(channel, page = 1) {
+async function fetchLatest(channel, page = 1, context) {
 	if (!channel.parameters?.studioId) {
 		return null;
 	}
@@ -104,18 +107,18 @@ async function fetchLatest(channel, page = 1) {
 	const res = await unprint.get(url);
 
 	if (res.ok && res.data?.contents) {
-		return scrapeAll(res.data.contents, channel);
+		return scrapeAll(res.data.contents, channel, context);
 	}
 
 	return res.status;
 }
 
-async function fetchScene(url, entity, baseRelease, include) {
+async function fetchScene(url, entity, baseRelease, context) {
 	const entryId = new URL(url).pathname.match(/\/watch(?:-page)?\/(\d+)/)[1];
 	const res = await unprint.get(`https://pornbox.com/contents/${entryId}`);
 
 	if (res.ok) {
-		return scrapeScene(res.data, entity, include);
+		return scrapeScene(res.data, entity, context);
 	}
 
 	return res.status;

src/scrapers/releases.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const acam = require('./acam');
 const adultempire = require('./adultempire');
 const angelogodshackoriginal = require('./angelogodshackoriginal');
 // const archangel = require('./archangel');
@@ -31,6 +32,7 @@ const hush = require('./hush');
 const innofsin = require('./innofsin');
 const insex = require('./insex');
 const inthecrack = require('./inthecrack');
+const jaxslayher = require('./jaxslayher');
 const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
 const julesjordan = require('./julesjordan');
 const karups = require('./karups');
@@ -42,6 +44,7 @@ const pornbox = require('./pornbox');
 const littlecapricedreams = require('./littlecapricedreams');
 const loveherfilms = require('./loveherfilms');
 const bluedonkeymedia = require('./bluedonkeymedia');
+const manyvids = require('./manyvids');
 const mikeadriano = require('./mikeadriano');
 const aylo = require('./aylo');
 const missax = require('./missax');
@@ -96,7 +99,10 @@ module.exports = {
 	freeuse: teamskeet,
 	familystrokes: teamskeet,
 	// etc
+	acam,
 	analvids: pornbox,
+	pornbox,
+	cosplayground: aziani,
 	kellymadison,
 	kink,
 	'8kmembers': kellymadison,
@@ -142,9 +148,11 @@ module.exports = {
 	julesjordan,
 	karups,
 	// etc.
+	jaxslayher,
 	littlecapricedreams,
 	loveherfilms,
 	mamacitaz: porndoe,
+	manyvids,
 	mariskax,
 	mikeadriano,
 	missax,

src/scrapers/rickysroom.js

@@ -23,9 +23,13 @@ function scrapeScene(data, channel) {
 	release.tags = data.tags;
 
 	release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
-	release.photos = data.previews?.full
+
-		.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+	// inconsistent data structure. if there is an album, it's previews: { full: [...] }, if it's empty, it's previews: []
-		.concat(data.thumbs); // screenshots
+	if (Array.isArray(data.previews?.full)) {
+		release.photos = data.previews?.full
+			.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+			.concat(data.thumbs); // screenshots
+	}
 
 	release.trailer = data.trailer_url;
 	release.teaser = data.special_thumbnails;
@@ -36,30 +40,6 @@ function scrapeScene(data, channel) {
 	return release;
 }
 
-function scrapeProfile(data, scenes, entity) {
-	const profile = {};
-
-	profile.entryId = data.id;
-	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
-
-	profile.description = data.Bio || data.bio;
-	profile.birthPlace = data.Born || data.born;
-	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
-
-	profile.measurements = data.Measurements || data.Measurements;
-	profile.height = feetInchesToCm(data.Height || data.height);
-	profile.weight = lbsToKg(data.Weight || data.weight);
-
-	profile.eyes = data.Eyes || data.eyes;
-	profile.hairColor = data.Hair || data.hair;
-
-	profile.avatar = data.thumb;
-
-	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
-
-	return profile;
-}
-
 async function fetchLatest(channel, page = 1) {
 	const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
 	const res = await qu.get(url);
@@ -102,6 +82,30 @@ async function fetchScene(url, channel, baseRelease) {
 	return res.status;
 }
 
+function scrapeProfile(data, scenes, entity) {
+	const profile = {};
+
+	profile.entryId = data.id;
+	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
+
+	profile.description = data.Bio || data.bio;
+	profile.birthPlace = data.Born || data.born;
+	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
+
+	profile.measurements = data.Measurements || data.Measurements;
+	profile.height = feetInchesToCm(data.Height || data.height);
+	profile.weight = lbsToKg(data.Weight || data.weight);
+
+	profile.eyes = data.Eyes || data.eyes;
+	profile.hairColor = data.Hair || data.hair;
+
+	profile.avatar = data.thumb;
+
+	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
+
+	return profile;
+}
+
 async function fetchProfile({ slug }, entity) {
 	const url = `${entity.url}/models/${slug}`;
 	const res = await qu.get(url);

src/scrapers/teencoreclub.js

@@ -18,7 +18,7 @@ function scrapeScene(scene, channel) {
 	const release = {};
 
 	release.entryId = scene.id;
-	release.url = `${channel.url}/video/${scene.id}/${scene.slug}`;
+	release.url = `${channel.origin}/video/${scene.id}/${scene.slug}`;
 
 	release.title = pickLocale(scene.title);
 	release.description = pickLocale(scene.description);
@@ -62,7 +62,8 @@ function scrapeScene(scene, channel) {
 }
 
 async function fetchLatest(channel, page = 1, { parameters }) {
-	const url = `https://api.fundorado.com/api/videos/browse/labels/${parameters.legacySiteId}?page=${page}&sg=false&sort=release&video_type=scene&lang=en`;
+	// sg: show gay
+	const url = `https://api.fundorado.com/api/videos/browse/labels/${parameters.legacySiteId}?page=${page}&sg=true&sort=release&video_type=scene&lang=en`;
 	const res = await unprint.get(url);
 
 	if (res.ok && res.data?.videos?.data) {

src/scrapers/whalemember.js

@@ -10,9 +10,12 @@ const dateRegex = /\d{4}-\d{2}-\d{2}T/;
 function scrapeLatest(scenes, fullData, channel, parameters) {
 	return scenes.map(({ query }) => {
 		const release = {};
+		const origin = new URL(parameters.latest || channel.url).origin;
 
-		release.url = query.url('[href*="/video"]', { origin: new URL(parameters.latest || channel.url).origin });
+		release.url = query.url('a[href*="/video"]', { origin })
-		release.title = query.content('a[href*="/video"] strong');
+			|| query.url('nuxtlink[to*="/video"]', { attribute: 'to', origin });
+
+		release.title = query.content('a[href*="/video"] strong, nuxtlink[to*="/video"] strong');
 
 		release.entryId = release.url
 			? new URL(release.url).pathname.split('/').at(-1)

src/store-releases.js

@@ -17,8 +17,6 @@ const { associateActors, associateDirectors, scrapeActors, toBaseActors } = requ
 const { associateReleaseTags } = require('./tags');
 const { curateEntity } = require('./entities');
 const { associateReleaseMedia } = require('./media');
-const { updateSceneSearch, updateMovieSearch } = require('./update-search');
-const { notify } = require('./alerts');
 
 const geoCommon = import('../common/geo.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
 
@@ -344,6 +342,9 @@ async function associateMovieScenes(movies, movieScenes) {
 		.filter(Boolean);
 
 	await bulkInsert('movies_scenes', associations, false);
+
+	// movieScenes includes scenes that are not associated to any movies, don't update those (again)
+	// await updateSceneSearch(associations.map((association) => association.scene_id));
 }
 
 async function associateSerieScenes(series, serieScenes) {
@@ -390,7 +391,7 @@ async function storeMovies(movies, useBatchId) {
 	const moviesWithId = attachReleaseIds(movies, storedMovies);
 
 	await associateReleaseTags(moviesWithId, 'movie');
-	await updateMovieSearch(moviesWithId.map((movie) => movie.id));
+	// await updateMovieSearch(moviesWithId.map((movie) => movie.id));
 
 	await associateReleaseMedia(moviesWithId, 'movie');
 
@@ -418,7 +419,8 @@ async function storeSeries(series, useBatchId) {
 	const storedSeries = await bulkInsert('series', curatedSerieEntries, ['entity_id', 'entry_id'], true);
 	const seriesWithId = attachReleaseIds(series, storedSeries);
 
-	await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
+	// TODO: the movie search function doesn't accept a second parameter, and we don't currently have a series search index at all; re-evaluate what actually needs to happen here
+	// await updateMovieSearch(seriesWithId.map((serie) => serie.id), 'serie');
 	await associateReleaseMedia(seriesWithId, 'serie');
 
 	return seriesWithId;
@@ -488,7 +490,7 @@ async function storeScenes(releases, useBatchId) {
 		await associateDirectors(releasesWithId, batchId); // some directors may also be actors, don't associate at the same time
 	}
 
-	await updateSceneSearch(releasesWithId.map((release) => release.id));
+	// await updateSceneSearch(releasesWithId.map((release) => release.id));
 
 	// media is more error-prone, associate separately
 	await associateReleaseMedia(releasesWithId);
@@ -501,8 +503,6 @@ async function storeScenes(releases, useBatchId) {
 
 	logger.info(`Stored ${storedReleaseEntries.length}, updated ${updated} releases`);
 
-	await notify(releasesWithId);
-
 	return releasesWithId;
 }
 
@@ -510,6 +510,4 @@ module.exports = {
 	associateMovieScenes,
 	storeScenes,
 	storeMovies,
-	updateSceneSearch,
-	updateMovieSearch,
 };

src/tools/biometrics.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const yargs = require('yargs');
+const { hideBin } = require('yargs/helpers');
+
+const knex = require('../knex');
+const { setBiometrics } = require('../biometrics');
+
+const argv = yargs(hideBin(process.argv))
+	.option('actor-ids', {
+		alias: 'actors',
+		type: 'array',
+		describe: 'List of actor IDs to derive biometrics for',
+	})
+	.option('update', {
+		alias: 'force',
+		type: 'boolean',
+		describe: 'Re-compute biometrics when already present',
+	})
+	.option('photos', {
+		type: 'boolean',
+		describe: 'Include secondary actor photos',
+	})
+	.argv;
+
+async function init() {
+	await setBiometrics(argv.actorIds, argv.update, argv.photos);
+	knex.destroy();
+}
+
+init();

src/tools/manticore-actors.js

@@ -1,107 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-
-const knex = require('../knex');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const searchApi = new manticore.SearchApi(mantiClient);
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchActors() {
-	// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
-	const actors = await knex.raw(`
-        SELECT
-			actors.*,
-			actors_meta.*,
-			date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
-        FROM actors
-		LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id
-	`);
-
-	return actors.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists actors');
-		await utilsApi.sql(`create table actors(
-			 id int,
-			 name text,
-			 slug string,
-			 gender string,
-			 date_of_birth timestamp,
-			 country string,
-			 has_avatar bool,
-			 mass int,
-			 height int,
-			 cup string,
-			 natural_boobs int,
-			 penis_length int,
-			 penis_girth int,
-			 stashed int,
-			 scenes int
-		) min_prefix_len = '3'`);
-
-		const actors = await fetchActors();
-
-		const docs = actors.map((actor) => ({
-			insert: {
-				index: 'actors',
-				id: actor.id,
-				doc: {
-					name: actor.name,
-					slug: actor.slug,
-					gender: actor.gender || undefined,
-					date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
-					has_avatar: !!actor.avatar_media_id,
-					country: actor.birth_country_alpha2 || undefined,
-					height: actor.height || undefined,
-					mass: actor.weight || undefined, // weight is a reserved keyword in manticore
-					cup: actor.cup || undefined,
-					natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
-					penis_length: actor.penis_length || undefined,
-					penis_girth: actor.penis_girth || undefined,
-					stashed: actor.stashed || 0,
-					scenes: actor.scenes || 0,
-				},
-			},
-		}));
-
-		const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
-			console.log(error);
-		});
-
-		console.log('data', data);
-		knex.destroy();
-
-		return;
-	}
-
-	const result = await searchApi.search({
-		index: 'actors',
-		query: {
-			equals: {
-				has_avatar: 1,
-			},
-		},
-		limit: 3,
-		sort: [{ slug: 'asc' }],
-	});
-
-	console.log(result);
-	console.log(result.hits?.hits);
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-movies.js

@@ -1,165 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-const { format } = require('date-fns');
-
-const knex = require('../knex');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-// const searchApi = new manticore.SearchApi(mantiClient);
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchMovies() {
-	const movies = await knex.raw(`
-        SELECT
-            movies.id AS id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			movies_covers IS NOT NULL as has_cover,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
-			row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
-        FROM movies
-		LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
-		LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
-		LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
-        LEFT JOIN entities ON movies.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
-        LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
-        LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
-        GROUP BY
-            movies.id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			movies_meta.stashed_scenes,
-			movies_meta.stashed_total,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			movies_covers.*
-	`);
-
-	return movies.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists movies');
-		await utilsApi.sql(`create table movies (
-			 id int,
-			 title text,
-			 title_filtered text,
-			 channel_id int,
-			 channel_name text,
-			 channel_slug text,
-			 network_id int,
-			 network_name text,
-			 network_slug text,
-			 entity_ids multi,
-			 actor_ids multi,
-			 actors text,
-			 tag_ids multi,
-			 tags text,
-			 meta text,
-			 date timestamp,
-			 has_cover bool,
-			 created_at timestamp,
-			 effective_date timestamp,
-			 stashed int,
-			 stashed_scenes int,
-			 stashed_total int,
-			 dupe_index int
-		)`);
-
-		const movies = await fetchMovies();
-
-		console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index));
-
-		const docs = movies.map((movie) => {
-			const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
-				id: tag.f1,
-				name: tag.f2,
-				priority: tag.f3,
-				alias: tag.f4,
-			}])));
-
-			const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-			const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-			const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
-
-			return {
-				replace: {
-					index: 'movies',
-					id: movie.id,
-					doc: {
-						title: movie.title || undefined,
-						title_filtered: filteredTitle || undefined,
-						date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
-						created_at: Math.round(movie.created_at.getTime() / 1000),
-						effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
-						channel_id: movie.channel_id,
-						channel_slug: movie.channel_slug,
-						channel_name: movie.channel_name,
-						network_id: movie.network_id || undefined,
-						network_slug: movie.network_slug || undefined,
-						network_name: movie.network_name || undefined,
-						entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
-						actor_ids: movie.actors.map((actor) => actor.f1),
-						actors: movie.actors.map((actor) => actor.f2).join(),
-						tag_ids: combinedTags.map((tag) => tag.id),
-						tags: flatTags.join(' '),
-						has_cover: movie.has_cover,
-						meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined,
-						stashed: movie.stashed || 0,
-						stashed_scenes: movie.stashed_scenes || 0,
-						stashed_total: movie.stashed_total || 0,
-						dupe_index: movie.dupe_index || 0,
-					},
-				},
-			};
-		});
-
-		console.log(docs.map((doc) => doc.replace));
-
-		const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-
-		console.log('data', data);
-	}
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-scenes.js

@@ -1,228 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-const { format } = require('date-fns');
-
-const knex = require('../knex');
-const chunk = require('../utils/chunk');
-const filterTitle = require('../utils/filter-title');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchScenes() {
-	const scenes = await knex.raw(`
-        SELECT
-            releases.id AS id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.entry_id,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			entities.alias as channel_aliases,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			parents.alias as network_aliases,
-			studios.id as studio_id,
-			studios.slug as studio_slug,
-			studios.name as studio_name,
-			grandparents.id as parent_network_id,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
-			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
-			COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
-			studios.showcased IS NOT false
-			AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
-			AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
-			AND (releases_summaries.batch_showcased IS NOT false)
-			AS showcased,
-			row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
-        FROM releases
-		LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
-		LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
-        LEFT JOIN entities AS studios ON studios.id = releases.studio_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
-        LEFT JOIN movies ON movies.id = movies_scenes.movie_id
-        LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
-        LEFT JOIN series ON series.id = series_scenes.serie_id
-        GROUP BY
-            releases.id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.entry_id,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			releases_summaries.batch_showcased,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			grandparents.id,
-			studios.id,
-			studios.name,
-			studios.slug,
-			entities.showcased,
-			parents.showcased,
-			studios.showcased;
-	`);
-
-	return scenes.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists scenes');
-		await utilsApi.sql(`create table scenes (
-			 id int,
-			 title text,
-			 title_filtered text,
-			 entry_id text,
-			 shoot_id text,
-			 channel_id int,
-			 channel_name text,
-			 channel_slug text,
-			 network_id int,
-			 network_name text,
-			 network_slug text,
-			 studio_id int,
-			 studio_name text,
-			 studio_slug text,
-			 entity_ids multi,
-			 actor_ids multi,
-			 actors text,
-			 tag_ids multi,
-			 tags text,
-			 movie_ids multi,
-			 movies text,
-			 serie_ids multi,
-			 series text,
-			 meta text,
-			 date timestamp,
-			 fingerprints text,
-			 is_showcased bool,
-			 created_at timestamp,
-			 effective_date timestamp,
-			 stashed int,
-			 dupe_index int
-		)`);
-
-		await utilsApi.sql('drop table if exists scenes_tags');
-		await utilsApi.sql(`create table scenes_tags (
-			 id int,
-			 scene_id int,
-			 tag_id int,
-			 actor_id int
-		)`);
-
-		console.log('Recreated scenes table');
-		console.log('Fetching scenes from primary database');
-
-		const scenes = await fetchScenes();
-
-		console.log('Fetched scenes from primary database');
-
-		const docs = scenes.flatMap((scene) => {
-			const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-			const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-			const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
-
-			return [
-				{
-					replace: {
-						index: 'scenes',
-						id: scene.id,
-						doc: {
-							title: scene.title || undefined,
-							title_filtered: filteredTitle || undefined,
-							date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-							created_at: Math.round(scene.created_at.getTime() / 1000),
-							effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-							is_showcased: scene.showcased,
-							entry_id: scene.entry_id || undefined,
-							shoot_id: scene.shoot_id || undefined,
-							channel_id: scene.channel_id,
-							channel_slug: scene.channel_slug,
-							channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
-							network_id: scene.network_id || undefined,
-							network_slug: scene.network_slug || undefined,
-							network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
-							studio_id: scene.studio_id || undefined,
-							studio_slug: scene.studio_slug || undefined,
-							studio_name: scene.studio_name || undefined,
-							entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-							actor_ids: scene.actors.map((actor) => actor.f1),
-							actors: scene.actors.map((actor) => actor.f2).join(),
-							tag_ids: scene.tags.map((tag) => tag.f1),
-							tags: flatTags.join(' '),
-							movie_ids: scene.movies.map((movie) => movie.f1),
-							movies: scene.movies.map((movie) => movie.f2).join(' '),
-							serie_ids: scene.series.map((serie) => serie.f1),
-							series: scene.series.map((serie) => serie.f2).join(' '),
-							fingerprints: scene.fingerprints.join(' '),
-							meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
-							stashed: scene.stashed || 0,
-							dupe_index: scene.dupe_index || 0,
-						},
-					},
-				},
-				...scene.tags.map((tag) => ({
-					replace: {
-						index: 'scenes_tags',
-						// id: scene.id,
-						doc: {
-							scene_id: scene.id,
-							tag_id: tag.f1,
-							actor_id: tag.f5,
-						},
-					},
-				})),
-			];
-		});
-
-		// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
-		chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
-			const acc = await chain;
-			const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
-
-			console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
-
-			return acc.concat(data.items);
-		}, Promise.resolve([]));
-
-		// console.log('data', accData);
-	}
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-stashes.js

@@ -1,88 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-
-const knex = require('../knex');
-const chunk = require('../utils/chunk');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-async function syncStashes(domain = 'scene') {
-	await utilsApi.sql(`truncate table ${domain}s_stashed`);
-
-	const stashes = await knex(`stashes_${domain}s`)
-		.select(
-			`stashes_${domain}s.id as stashed_id`,
-			`stashes_${domain}s.${domain}_id`,
-			'stashes.id as stash_id',
-			'stashes.user_id as user_id',
-			`stashes_${domain}s.created_at as created_at`,
-		)
-		.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
-
-	await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
-		await chain;
-
-		const stashDocs = stashChunk.map((stash) => ({
-			replace: {
-				index: `${domain}s_stashed`,
-				id: stash.stashed_id,
-				doc: {
-					[`${domain}_id`]: stash[`${domain}_id`],
-					stash_id: stash.stash_id,
-					user_id: stash.user_id,
-					created_at: Math.round(stash.created_at.getTime() / 1000),
-				},
-			},
-		}));
-
-		await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
-
-		console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
-	}, Promise.resolve());
-}
-
-async function init() {
-	await utilsApi.sql('drop table if exists scenes_stashed');
-
-	await utilsApi.sql(`create table if not exists scenes_stashed (
-		scene_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await utilsApi.sql('drop table if exists movies_stashed');
-
-	await utilsApi.sql(`create table if not exists movies_stashed (
-		movie_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await utilsApi.sql('drop table if exists actors_stashed');
-
-	await utilsApi.sql(`create table if not exists actors_stashed (
-		actor_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await syncStashes('scene');
-	await syncStashes('actor');
-	await syncStashes('movie');
-
-	console.log('Done!');
-
-	knex.destroy();
-}
-
-init();

src/tools/update-search.js

@@ -1,12 +0,0 @@
-'use strict';
-
-const { updateSceneSearch, updateMovieSearch } = require('../update-search');
-
-async function init() {
-	await updateSceneSearch();
-	await updateMovieSearch();
-
-	process.exit();
-}
-
-init();

src/update-search.js

@@ -1,436 +1,48 @@
 'use strict';
 
-const manticore = require('manticoresearch');
+const config = require('config');
-const { format } = require('date-fns');
+const unprint = require('unprint');
 
 const knex = require('./knex');
-const logger = require('./logger')(__filename);
-const bulkInsert = require('./utils/bulk-insert');
-const chunk = require('./utils/chunk');
-const filterTitle = require('./utils/filter-title');
 
-const mantiClient = new manticore.ApiClient();
+async function syncWeb(domain, ids) {
-const indexApi = new manticore.IndexApi(mantiClient);
+	if (!ids || ids.length === 0) {
-
-async function updateManticoreStashedScenes(docs) {
-	await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
-		await chain;
-
-		const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
-
-		const stashes = await knex('stashes_scenes')
-			.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
-			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
-			.whereIn('scene_id', sceneIds);
-
-		const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
-			const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
-
-			if (sceneStashes.length === 0) {
-				return [];
-			}
-
-			const stashDoc = sceneStashes.map((stash) => ({
-				replace: {
-					index: 'scenes_stashed',
-					id: stash.stashed_id,
-					doc: {
-						// ...doc.replace.doc,
-						scene_id: doc.replace.id,
-						user_id: stash.user_id,
-						stash_id: stash.stash_id,
-						created_at: Math.round(stash.created_at.getTime() / 1000),
-					},
-				},
-			}));
-
-			return stashDoc;
-		});
-
-		if (stashDocs.length > 0) {
-			await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
-		}
-
-		const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
-
-		if (deleteSceneIds.length > 0) {
-			await indexApi.callDelete({
-				index: 'scenes_stashed',
-				query: {
-					bool: {
-						must: [
-							{
-								in: {
-									scene_id: deleteSceneIds,
-								},
-							},
-						],
-					},
-				},
-			});
-		}
-	}, Promise.resolve());
-}
-
-async function updateManticoreSceneSearch(releaseIds) {
-	logger.info(`Updating Manticore search documents for ${releaseIds ? releaseIds.length : 'all' } scenes`);
-
-	const scenes = await knex.raw(`
-        SELECT
-            releases.id AS id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			studios.id as studio_id,
-			studios.slug as studio_slug,
-			studios.name as studio_name,
-			grandparents.id as parent_network_id,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
-			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
-			studios.showcased IS NOT false
-			AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
-			AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
-			AND (releases_summaries.batch_showcased IS NOT false)
-			AS showcased,
-			row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
-        FROM releases
-		LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
-		LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
-        LEFT JOIN entities AS studios ON studios.id = releases.studio_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
-        LEFT JOIN movies ON movies.id = movies_scenes.movie_id
-        LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
-        LEFT JOIN series ON series.id = series_scenes.serie_id
-        ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
-        GROUP BY
-            releases.id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			releases_summaries.batch_showcased,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			entities.showcased,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			grandparents.id,
-			studios.id,
-			studios.name,
-			studios.slug,
-			parents.showcased,
-			studios.showcased
-    `, releaseIds && [releaseIds]);
-
-	const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
-
-	const docs = releaseIds.map((sceneId) => {
-		const scene = scenesById[sceneId];
-
-		if (!scene) {
-			return {
-				delete: {
-					index: 'scenes',
-					id: sceneId,
-				},
-			};
-		}
-
-		const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
-		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
-		const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
-
-		return {
-			replace: {
-				index: 'scenes',
-				id: scene.id,
-				doc: {
-					title: scene.title || undefined,
-					title_filtered: filteredTitle || undefined,
-					date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-					created_at: Math.round(scene.created_at.getTime() / 1000),
-					effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-					is_showcased: scene.showcased,
-					shoot_id: scene.shoot_id || undefined,
-					channel_id: scene.channel_id,
-					channel_slug: scene.channel_slug,
-					channel_name: scene.channel_name,
-					network_id: scene.network_id || undefined,
-					network_slug: scene.network_slug || undefined,
-					network_name: scene.network_name || undefined,
-					studio_id: scene.studio_id || undefined,
-					studio_slug: scene.studio_slug || undefined,
-					studio_name: scene.studio_name || undefined,
-					entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-					actor_ids: scene.actors.map((actor) => actor.f1),
-					actors: scene.actors.map((actor) => actor.f2).join(),
-					tag_ids: scene.tags.map((tag) => tag.f1),
-					tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
-					movie_ids: scene.movies.map((movie) => movie.f1),
-					movies: scene.movies.map((movie) => movie.f2).join(' '),
-					serie_ids: scene.series.map((serie) => serie.f1),
-					series: scene.series.map((serie) => serie.f2).join(' '),
-					meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
-					stashed: scene.stashed || 0,
-					dupe_index: scene.dupe_index || 0,
-				},
-			},
-		};
-	});
-
-	if (docs.length === 0) {
 		return;
 	}
 
-	await Promise.all([
+	await knex('sync').insert({ domain, item_ids: Array.from(new Set(ids)) });
-		indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')),
-		updateManticoreStashedScenes(docs),
-	]);
-}
 
-async function updateSqlSceneSearch(releaseIds) {
+	if (config.webApi.enabled) {
-	logger.info(`Updating SQL search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
+		await unprint.post(`${config.webApi.address}/sync`, null, {
-
+			headers: {
-	const documents = await knex.raw(`
+				'api-user': config.webApi.apiUserId,
-        SELECT
+				'api-key': config.webApi.apiKey,
-            releases.id AS release_id,
+			},
-            TO_TSVECTOR(
+		});
-                'english',
-                COALESCE(releases.title, '') || ' ' ||
-                releases.entry_id || ' ' ||
-                entities.name || ' ' ||
-                entities.slug || ' ' ||
-                COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
-                COALESCE(parents.name, '') || ' ' ||
-                COALESCE(parents.slug, '') || ' ' ||
-                COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
-                COALESCE(releases.shoot_id, '') || ' ' ||
-                COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
-                STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
-            ) as document
-        FROM releases
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
-        GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
-    `, releaseIds && [releaseIds]);
-
-	if (documents.rows?.length > 0) {
-		await bulkInsert('releases_search', documents.rows, ['release_id']);
 	}
-
-	await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
 }
 
 async function updateSceneSearch(releaseIds) {
 	await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
+	await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
 
-	await updateSqlSceneSearch(releaseIds);
+	await syncWeb('scene', releaseIds);
-	await updateManticoreSceneSearch(releaseIds);
-}
-
-async function updateManticoreMovieSearch(movieIds) {
-	logger.info(`Updating Manticore search documents for ${movieIds ? movieIds.length : 'all' } movies`);
-
-	const movies = await knex.raw(`
-        SELECT
-            movies.id AS id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			movies_covers IS NOT NULL as has_cover,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
-			row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
-        FROM movies
-		LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
-		LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
-		LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
-        LEFT JOIN entities ON movies.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
-        LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
-        LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
-        ${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
-        GROUP BY
-            movies.id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			movies_meta.stashed_scenes,
-			movies_meta.stashed_total,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			movies_covers.*
-    `, movieIds && [movieIds]);
-
-	const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
-
-	const docs = movieIds.map((movieId) => {
-		const movie = moviesById[movieId];
-
-		if (!movie) {
-			return {
-				delete: {
-					index: 'movies',
-					id: movieId,
-				},
-			};
-		}
-
-		const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
-			id: tag.f1,
-			name: tag.f2,
-			priority: tag.f3,
-			alias: tag.f4,
-		}])));
-
-		const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-		const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-		const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
-
-		return {
-			replace: {
-				index: 'movies',
-				id: movie.id,
-				doc: {
-					title: movie.title || undefined,
-					title_filtered: filteredTitle || undefined,
-					date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
-					created_at: Math.round(movie.created_at.getTime() / 1000),
-					effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
-					channel_id: movie.channel_id,
-					channel_slug: movie.channel_slug,
-					channel_name: movie.channel_name,
-					network_id: movie.network_id || undefined,
-					network_slug: movie.network_slug || undefined,
-					network_name: movie.network_name || undefined,
-					entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
-					actor_ids: movie.actors.map((actor) => actor.f1),
-					actors: movie.actors.map((actor) => actor.f2).join(),
-					tag_ids: combinedTags.map((tag) => tag.id),
-					tags: flatTags.join(' '),
-					has_cover: movie.has_cover,
-					meta: movie.date ? format(movie.date, 'y yy M MMM MMMM d') : undefined,
-					stashed: movie.stashed || 0,
-					stashed_scenes: movie.stashed_scenes || 0,
-					stashed_total: movie.stashed_total || 0,
-					dupe_index: movie.dupe_index || 0,
-				},
-			},
-		};
-	});
-
-	if (docs.length === 0) {
-		return;
-	}
-
-	await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-}
-
-async function updateSqlMovieSearch(movieIds, target = 'movie') {
-	logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
-
-	const documents = await knex.raw(`
-		SELECT
-			${target}s.id AS ${target}_id,
-				TO_TSVECTOR(
-				'english',
-					COALESCE(${target}s.title, '') || ' ' ||
-					entities.name || ' ' ||
-					entities.slug || ' ' ||
-					COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
-					COALESCE(parents.name, '') || ' ' ||
-					COALESCE(parents.slug, '') || ' ' ||
-					COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
-					COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
-					STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(tags.name, ''), ' ')
-			) as document
-		FROM ${target}s
-		LEFT JOIN entities ON ${target}s.entity_id = entities.id
-		LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-		LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
-		LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
-		LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
-		LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
-		LEFT JOIN actors ON actors.id = releases_actors.actor_id
-		LEFT JOIN tags ON tags.id = releases_tags.tag_id
-        ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
-		GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
-    `, movieIds && [movieIds]);
-
-	if (documents.rows?.length > 0) {
-		await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
-	}
 }
 
 async function updateMovieSearch(releaseIds) {
 	await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;');
 
-	await updateSqlMovieSearch(releaseIds);
+	await syncWeb('movie', releaseIds);
-	await updateManticoreMovieSearch(releaseIds);
+}
+
+async function updateActorSearch(actorIds) {
+	await knex.raw('REFRESH MATERIALIZED VIEW actors_meta;');
+
+	await syncWeb('actor', actorIds);
 }
 
 module.exports = {
 	updateSceneSearch,
 	updateMovieSearch,
+	updateActorSearch,
 };

src/utils/http.js

@@ -8,7 +8,6 @@ const fs = require('fs').promises;
 // const util = require('util');
 // const stream = require('stream');
 const { pipeline } = require('stream/promises');
-const tunnel = require('tunnel');
 const Bottleneck = require('bottleneck');
 const { JSDOM, toughCookie } = require('jsdom');
 const puppeteer = require('puppeteer-extra');
@@ -328,11 +327,8 @@ const defaultAgent = new undici.Agent({
 	},
 });
 
-const proxyAgent = tunnel.httpsOverHttp({
+const proxyAgent = new undici.ProxyAgent({
-	proxy: {
+	uri: `http://${config.proxy.host}:${config.proxy.port}`,
-		host: config.proxy.host,
-		port: config.proxy.port,
-	},
 });
 
 async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
@@ -438,8 +434,8 @@ async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
 		...defaultOptions,
 		...requestOptions,
 		headers: {
-			...(requestOptions.includeDefaultHeaders === false ? {} : defaultOptions.headers),
+			...Object.fromEntries(requestOptions.includeDefaultHeaders === false ? {} : Object.entries(defaultOptions.headers).map(([key, value]) => [key.toLowerCase(), value])),
-			...requestOptions.headers,
+			...Object.fromEntries(Object.entries(requestOptions.headers || {}).map(([key, value]) => [key.toLowerCase(), value])),
 		},
 		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
 		stream: !!requestOptions.destination,

tests/profiles.js

@@ -213,7 +213,7 @@ const actors = [
 	{ entity: 'naughtyamerica', name: 'Nicole Aniston', fields: ['avatar', 'description'] },
 	{ entity: 'tonightsgirlfriend', name: 'Abella Danger', fields: ['avatar'] },
 	// jules jordan scraper
-	{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'description', 'avatar'] },
+	{ entity: 'julesjordan', name: 'Vanna Bardot', fields: ['height', 'dateOfBirth', 'measurements', 'avatar'] },
 	{ entity: 'amateurallure', name: 'Ava Amira', fields: ['avatar', 'description'] },
 	{ entity: 'swallowsalon', name: 'Abella Danger', fields: ['avatar'] },
 	// exploitedx
@@ -265,6 +265,8 @@ const actors = [
 	{ entity: 'tokyohot', name: 'Mai Kawana', url: 'https://my.tokyo-hot.com/cast/2099/', fields: ['avatar', 'birthPlace', 'height', 'cup', 'bust', 'waist', 'hip', 'hairStyle', 'shoeSize', 'bloodType'] },
 	{ entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
 	{ entity: 'darkkotv', name: 'Aidra Fox', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'measurements', 'naturalBoobs', 'hasTattoos', 'hasPiercings'] },
+	{ entity: 'jaxslayher', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'eyes', 'height', 'weight', 'birthPlace', 'measurements', 'hairColor'] },
+	{ entity: 'cosplayground', name: 'Anna Claire Clouds', url: 'https://cosplayground.com/models/anna-claire-clouds?models=Anna%20Claire%20Clouds', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'height', 'measurements', 'hairColor', 'ethnicity'] },
 ];
 
 const actorScrapers = scrapers.actors;