Updating Manticore scenes after tag reassociation.

.eslintrc

@@ -27,7 +27,7 @@
 		"require-await": "off",
         "no-param-reassign": ["error", {
             "props": true,
-            "ignorePropertyModificationsFor": ["state", "acc", "req"]
+            "ignorePropertyModificationsFor": ["state", "acc", "req", "error"]
         }]
     },
     "globals": {

config/default.js

@@ -188,6 +188,8 @@ module.exports = {
 			'wishescumtrue',
 			// hentaied
 			'somegore',
+			// digital playground
+			'digitalplayground', // no longer updates, produces a bunch of garbage for some reason
 		],
 		networks: [
 			// dummy network for testing
@@ -279,7 +281,7 @@ module.exports = {
 		trailerQuality: [540, 720, 960, 480, 1080, 360, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
 		limit: 25, // max number of photos per release
 		attempts: 2,
-		flushOrphaned: true,
+		flushOrphaned: false,
 		flushWindow: 1000,
 		streams: {
 			enabled: true, // fetch streams

migrations/20260302222545_series_alt_descriptions.js

@@ -0,0 +1,13 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('series', (table) => {
+		table.specificType('alt_descriptions', 'text ARRAY');
+		table.json('attributes');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('series', (table) => {
+		table.dropColumn('alt_descriptions');
+		table.dropColumn('attributes');
+	});
+};

migrations/20260304020542_scene_actor_tags.js

@@ -0,0 +1,21 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('releases_tags', (table) => {
+		table.integer('actor_id')
+			.references('id')
+			.inTable('actors');
+
+		table.dropUnique(['tag_id', 'release_id']);
+	});
+
+	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('releases_tags', (table) => {
+		table.dropColumn('actor_id');
+
+		table.unique(['tag_id', 'release_id']);
+	});
+
+	await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
+};

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.250.12",
+    "version": "1.250.42",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.250.12",
+            "version": "1.250.42",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.250.12",
+    "version": "1.250.42",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/00_tags.js

@@ -815,6 +815,10 @@ const tags = [
 		description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
 		group: 'group',
 	},
+	{
+		name: 'spitroast',
+		slug: 'spitroast',
+	},
 	{
 		name: 'military',
 		slug: 'military',
@@ -1893,6 +1897,14 @@ const aliases = [
 		name: 'double penetration (dp)',
 		for: 'dp',
 	},
+	{
+		name: 'double penetration ass pussy',
+		for: 'dp',
+	},
+	{
+		name: 'double penetration mouth pussy',
+		for: 'spitroast',
+	},
 	{
 		name: 'double penetration - dp',
 		for: 'dp',
@@ -2355,11 +2367,6 @@ const aliases = [
 		name: 'spit',
 		for: 'saliva',
 	},
-	{
-		name: 'spitroast',
-		for: 'mfm',
-		secondary: true,
-	},
 	{
 		name: 'spoon',
 		for: 'spooning',
@@ -3019,11 +3026,13 @@ const priorities = [ // higher index is higher priority
 	['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
 	['lesbian', 'rough', 'milf', 'male-focus', 'bdsm', 'oil'],
 	['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
-	['anal', 'bukkake'],
+	['anal', 'bukkake', 'spitroast'],
 	['dp', 'dap', 'triple-penetration', 'tap', 'dvp', 'tvp', 'airtight'],
 	['blowbang', 'orgy'],
 	['gangbang'],
 	['gay', 'transsexual', 'bisexual', 'hentai'],
+	['pissing'],
+	['compilation', 'bts'],
 ].reduce((acc, slugs, index) => {
 	slugs.forEach((slug) => { acc[slug] = index; });
 

seeds/01_networks.js

@@ -2,6 +2,14 @@
 const omit = require('object.omit');
 
 const upsert = require('../src/utils/upsert');
+const redis = require('../src/redis');
+
+const entityPrefixes = {
+	channel: '',
+	network: '_',
+	studio: '*',
+	info: '@',
+};
 
 const grandParentNetworks = [
 	{
@@ -905,80 +913,88 @@ const networks = [
 	},
 ];
 
-exports.seed = (knex) => Promise.resolve()
-	.then(async () => {
-		await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => {
-			if (network.rename) {
-				return knex('entities')
-					.where({
-						type: network.type || 'network',
-						slug: network.rename,
-					})
-					.update('slug', network.slug);
-			}
+exports.seed = async (knex) => {
+	await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => {
+		if (network.rename) {
+			return knex('entities')
+				.where({
+					type: network.type || 'network',
+					slug: network.rename,
+				})
+				.update('slug', network.slug);
+		}
 
-			return null;
-		}).filter(Boolean));
+		return null;
+	}).filter(Boolean));
 
-		const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex);
-		const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
+	const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex);
+	const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
 
-		const parentNetworksWithGrandParent = parentNetworks.map((network) => ({
-			slug: network.slug,
-			name: network.name,
-			type: network.type || 'network',
-			alias: network.alias,
-			url: network.url,
-			description: network.description,
-			has_logo: network.hasLogo ?? true,
-			showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
-			parameters: network.parameters || null,
-			options: network.options,
-			parent_id: grandParentNetworksBySlug[network.parent] || null,
-		}));
+	const parentNetworksWithGrandParent = parentNetworks.map((network) => ({
+		slug: network.slug,
+		name: network.name,
+		type: network.type || 'network',
+		alias: network.alias,
+		url: network.url,
+		description: network.description,
+		has_logo: network.hasLogo ?? true,
+		showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
+		parameters: network.parameters || null,
+		options: network.options,
+		parent_id: grandParentNetworksBySlug[network.parent] || null,
+	}));
 
-		const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex);
-		const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
+	const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex);
+	const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
 
-		const networksWithParent = networks.map((network) => ({
-			slug: network.slug,
-			name: network.name,
-			type: network.type || 'network',
-			alias: network.alias,
-			url: network.url,
-			description: network.description,
-			has_logo: network.hasLogo ?? true,
-			showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
-			parameters: network.parameters || null,
-			options: network.options,
-			parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
-		}));
+	const networksWithParent = networks.map((network) => ({
+		slug: network.slug,
+		name: network.name,
+		type: network.type || 'network',
+		alias: network.alias,
+		url: network.url,
+		description: network.description,
+		has_logo: network.hasLogo ?? true,
+		showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
+		parameters: network.parameters || null,
+		options: network.options,
+		parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
+	}));
 
-		const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex);
+	const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex);
 
-		const networkIdsBySlug = [].concat(
-			grandParentNetworkEntries.inserted,
-			grandParentNetworkEntries.updated,
-			parentNetworkEntries.inserted,
-			parentNetworkEntries.updated,
-			networkEntries.inserted,
-			networkEntries.updated,
-		).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
+	const networkIdsBySlug = [].concat(
+		grandParentNetworkEntries.inserted,
+		grandParentNetworkEntries.updated,
+		parentNetworkEntries.inserted,
+		parentNetworkEntries.updated,
+		networkEntries.inserted,
+		networkEntries.updated,
+	).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
 
-		const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean);
+	const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean);
 
-		const tagEntries = await knex('tags').whereIn('slug', tagSlugs);
-		const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
+	const tagEntries = await knex('tags').whereIn('slug', tagSlugs);
+	const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
 
-		const tagAssociations = networks
-			.map((network) => (network.tags
-				? network.tags.map((tagSlug) => ({
-					entity_id: networkIdsBySlug[network.slug],
-					tag_id: tagIdsBySlug[tagSlug],
-					inherit: true,
-				}))
-				: []))
-			.flat();
+	const tagAssociations = networks
+		.map((network) => (network.tags
+			? network.tags.map((tagSlug) => ({
+				entity_id: networkIdsBySlug[network.slug],
+				tag_id: tagIdsBySlug[tagSlug],
+				inherit: true,
+			}))
+			: []))
+		.flat();
 
-		await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
-	});
+	await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
+
+	const entities = await knex('entities').select('id', 'slug', 'type');
+
+	await redis.connect();
+
+	await redis.del('traxxx:entities:id_by_slug');
+	await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
+
+	await redis.disconnect();
+};

seeds/02_sites.js

@@ -1,4 +1,12 @@
 const upsert = require('../src/utils/upsert');
+const redis = require('../src/redis');
+
+const entityPrefixes = {
+	channel: '',
+	network: '_',
+	studio: '*',
+	info: '@',
+};
 
 /* eslint-disable max-len */
 const sites = [
@@ -609,6 +617,9 @@ const sites = [
 		slug: 'asgmaxoriginals',
 		url: 'https://www.asgmax.com/en/channel/asgmaxoriginals',
 		parent: 'asgmax',
+		parameters: {
+			queryChannel: 'asgmaxoriginals',
+		},
 	},
 	{
 		name: 'ASG Max Films',
@@ -616,6 +627,9 @@ const sites = [
 		url: 'https://www.asgmax.com/en/channel/asgmaxfilms',
 		parent: 'asgmax',
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmaxfilms',
+		},
 	},
 	{
 		name: 'ASG International',
@@ -623,6 +637,9 @@ const sites = [
 		url: 'https://www.asgmax.com/en/channel/asginternational',
 		parent: 'asgmax',
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asginternational',
+		},
 	},
 	{
 		name: 'ASG Massage',
@@ -631,6 +648,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['massage'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmassage',
+		},
 	},
 	{
 		name: 'ASG Auditions',
@@ -639,6 +659,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['audition'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgauditions',
+		},
 	},
 	{
 		name: 'ASG Free Use',
@@ -647,6 +670,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['free-use'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgfreeuse',
+		},
 	},
 	{
 		name: 'Exeter Hill College',
@@ -655,6 +681,9 @@ const sites = [
 		parent: 'asgmax',
 		hasLogo: false,
 		tags: ['animated'],
+		parameters: {
+			queryChannel: 'asgexeterhillcollege',
+		},
 	},
 	// ASG MAX INDEPENDENT
 	{
@@ -1018,12 +1047,12 @@ const sites = [
 	{
 		name: 'Disruptive Films',
 		slug: 'disruptivefilms',
-		delete: true,
 		url: 'https://www.disruptivefilms.com',
 		parent: 'disruptivefilms',
 		tags: ['gay'],
 		parameters: {
 			queryChannel: 'asgmaxdisruptivefilms',
+			filterChannel: 'asgmaxdisruptivefilms',
 		},
 	},
 	{
@@ -1070,6 +1099,17 @@ const sites = [
 			queryChannel: 'asgmaxtruemale',
 		},
 	},
+	{
+		name: 'Deep Inside',
+		slug: 'deepinside',
+		url: 'https://www.asgmax.com/en/channel/asgmaxdeepinside',
+		parent: 'disruptivefilms',
+		tags: ['gay'],
+		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmaxdeepinside',
+		},
+	},
 	// AMATEUR ALLURE
 	{
 		name: 'Amateur Allure',
@@ -5422,6 +5462,12 @@ const sites = [
 			referer: 'https://www.girlsway.com',
 		},
 	},
+	{
+		slug: 'lesbianfactor',
+		name: 'Lesbian Factor',
+		url: 'https://www.lesbianfactor.com',
+		parent: 'girlsway',
+	},
 	// HITZEFREI
 	{
 		slug: 'unleashed',
@@ -5532,6 +5578,16 @@ const sites = [
 			latest: 'https://seehimfuck.com',
 		},
 	},
+	{
+		slug: 'seehimsolo',
+		name: 'See Him Solo',
+		url: 'https://seehimsolo.com',
+		tags: ['male-focus', 'solo'],
+		parent: 'hussiepass',
+		parameters: {
+			latest: 'https://seehimsolo.com/categories/movies-2/{page}/latest/',
+		},
+	},
 	{
 		slug: 'interracialpovs',
 		name: 'Interracial POVs',
@@ -9381,9 +9437,9 @@ const sites = [
 		parent: 'nubiles',
 	},
 	{
-		slug: 'caughtmycoach',
-		name: 'Caught My Coach',
-		url: 'https://caughtmycoach.com',
+		slug: 'shesbreedingmaterial',
+		name: 'She\'s Breeding Material',
+		url: 'https://shesbreedingmaterial.com',
 		parent: 'nubiles',
 	},
 	// PASCALS SUBSLUTS
@@ -10506,15 +10562,28 @@ const sites = [
 			siteAsSerie: true,
 		},
 	},
+	{
+		name: 'Hardwerk',
+		slug: 'hardwerk',
+		url: 'https://hardwerk.com',
+		independent: true,
+		parent: 'radical',
+		parameters: {
+			endpoint: 'jC4SrjH8YVDtRejiA0PMx',
+			videos: 'films',
+			actors: 'performers',
+		},
+	},
 	// REALITY KINGS
 	{
 		name: 'Look At Her Now',
 		url: 'https://www.lookathernow.com',
 		description: 'Look At Her Now brings you best HD reality porn videos every week. Check out these girls before and after they get some rough pounding.',
-		parameters: { native: true },
-		// parameters: { siteId: 300 },
 		slug: 'lookathernow',
 		parent: 'realitykings',
+		parameters: {
+			siteId: 364,
+		},
 	},
 	{
 		name: 'We Live Together',
@@ -15491,195 +15560,86 @@ sites.reduce((acc, site) => {
 }, new Set());
 
 /* eslint-disable max-len */
-exports.seed = (knex) => Promise.resolve()
-	.then(async () => {
-		await Promise.all(sites.map(async (channel) => {
-			if (channel.rename) {
-				await knex('entities')
-					.where({
-						type: channel.type || 'channel',
-						slug: channel.rename,
-					})
-					.update('slug', channel.slug);
+exports.seed = async (knex) => {
+	await Promise.all(sites.map(async (channel) => {
+		if (channel.rename) {
+			await knex('entities')
+				.where({
+					type: channel.type || 'channel',
+					slug: channel.rename,
+				})
+				.update('slug', channel.slug);
 
-				return;
+			return;
+		}
+
+		if (channel.delete) {
+			await knex('entities')
+				.where({
+					type: channel.type || 'channel',
+					slug: channel.slug,
+				})
+				.delete();
+		}
+	}).filter(Boolean));
+
+	const networks = await knex('entities')
+		.where('type', 'network')
+		.orWhereNull('parent_id');
+
+	const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+
+	const tags = await knex('tags').select('*').whereNull('alias_for');
+	const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+
+	const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
+		slug: site.slug,
+		name: site.name,
+		name_stylized: site.style,
+		type: site.type || 'channel',
+		alias: site.alias,
+		description: site.description,
+		url: site.url,
+		parameters: site.parameters || null,
+		options: site.options,
+		parent_id: networksMap[site.parent] || null,
+		priority: site.priority || 0,
+		independent: !!site.independent,
+		visible: site.visible,
+		showcased: site.showcased,
+		has_logo: site.hasLogo === undefined ? true : site.hasLogo,
+	}));
+
+	const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex);
+	const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+
+	const tagAssociations = sites.map((site) => (site.tags && !site.delete
+		? site.tags.map((tagSlug) => {
+			const tag = tagsMap[tagSlug];
+
+			if (!tag) {
+				console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
 			}
 
-			if (channel.delete) {
-				await knex('entities')
-					.where({
-						type: channel.type || 'channel',
-						slug: channel.slug,
-					})
-					.delete();
-			}
-		}).filter(Boolean));
+			return {
+				entity_id: sitesMap[site.slug],
+				tag_id: tagsMap[tagSlug],
+				inherit: true,
+			};
+		})
+		: []
+	)).flat();
 
-		const networks = await knex('entities')
-			.where('type', 'network')
-			.orWhereNull('parent_id');
+	await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
 
-		const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+	const entities = await knex('entities').select('id', 'slug', 'type');
 
-		const tags = await knex('tags').select('*').whereNull('alias_for');
-		const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
+	await redis.connect();
 
-		const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
-			slug: site.slug,
-			name: site.name,
-			name_stylized: site.style,
-			type: site.type || 'channel',
-			alias: site.alias,
-			description: site.description,
-			url: site.url,
-			parameters: site.parameters || null,
-			options: site.options,
-			parent_id: networksMap[site.parent],
-			priority: site.priority || 0,
-			independent: !!site.independent,
-			visible: site.visible,
-			showcased: site.showcased,
-			has_logo: site.hasLogo === undefined ? true : site.hasLogo,
-		}));
+	await redis.del('traxxx:entities:id_by_slug');
+	await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
 
-		const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex);
-		const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
-
-		const tagAssociations = sites.map((site) => (site.tags && !site.delete
-			? site.tags.map((tagSlug) => {
-				const tag = tagsMap[tagSlug];
-
-				if (!tag) {
-					console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
-				}
-
-				return {
-					entity_id: sitesMap[site.slug],
-					tag_id: tagsMap[tagSlug],
-					inherit: true,
-				};
-			})
-			: []
-		)).flat();
-
-		return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
-	});
+	await redis.disconnect();
+};
 
 exports.sites = sites;
-
-/*
-            'X-Art'               => 'xart',
-            'met-art'             => 'metart',
-            '18og'                => '18OnlyGirls',
-            'a1o1'                => 'Asian1on1',
-            'add'                 => 'ManualAddActors',
-            'analb'               => 'AnalBeauty',
-            'bgonzo'              => 'BangGonzo',
-            'btlbd'               => 'BigTitsLikeBigDicks',
-            'bjf'                 => 'BlowjobFridays',
-            'cws'                 => 'CzechWifeSwap',
-            'Daughter'            => 'DaughterSwap',
-            'Daughters'           => 'DaughterSwap',
-            'dc'                  => 'DorcelVision',
-            'dpg'                 => 'DigitalPlayground',
-            'dsw'                 => 'DaughterSwap',
-            'faq'                 => 'FirstAnalQuest',
-            'ft'                  => 'FastTimes',
-            'fittingroom'         => 'Fitting-Room',
-            'gbcp'                => 'GangbangCreampie',
-            'hart'                => 'Hegre',
-            'hegre-art'           => 'Hegre',
-            'kha'                 => 'KarupsHA',
-            'kow'                 => 'KarupsOW',
-            'kpc'                 => 'KarupsPC',
-            'la'                  => 'LatinAdultery',
-            'lcd'                 => 'LittleCaprice',
-            'lhf'                 => 'LoveHerFeet',
-            'littlecapricedreams' => 'Little Caprice Dreams',
-            'maj'                 => 'ManoJob',
-            'mfl'                 => 'Mofos',
-            'mj'                  => 'ManoJob',
-            'mpov'                => 'MrPOV',
-            'naughtyamericavr'    => 'NaughtyAmerica',
-            'news'                => 'NewSensations',
-            'ps'                  => 'PropertySex',
-            'sart'                => 'SexArt',
-            'sbj'                 => 'StreetBlowjobs',
-            'sislove'             => 'SisLovesMe',
-            'tds'                 => 'TheDickSuckers',
-            'these'               => 'TheStripperExperience',
-            'tlc'                 => 'TeensLoveCream',
-            'tle'                 => 'TheLifeErotic',
-            'tog'                 => 'TonightsGirlfriend',
-            'wowg'                => 'WowGirls',
-            'wy'                  => 'WebYoung',
-            'itc'                 => 'InTheCrack',
-            "abbw"                => "AbbyWinters",
-            "abme"                => "AbuseMe",
-            "ana"                 => "AnalAngels",
-            "atke"                => "ATKExotics",
-            "atkg"                => "ATKGalleria",
-            "atkgfs"              => "ATKGirlfriends",
-            "atkh"                => "ATKHairy",
-            "aktp"                => "ATKPetites",
-            "ba"                  => "Beauty-Angels",
-            "bna"                 => "BrandNew",
-            "bam"                 => "BruceAndMorgan",
-            "bcast"               => "BrutalCastings",
-            "bd"                  => "BrutalDildos",
-            "bpu"                 => "BrutalPickups",
-            "cza"                 => "CzhecAmateurs",
-            "czbb"                => "CzechBangBus",
-            "czb"                 => "CzechBitch",
-            "cc"                  => "CzechCasting",
-            "czc"                 => "CzechCouples",
-            "czestro"             => "CzechEstrogenolit",
-            "czf"                 => "CzechFantasy",
-            "czgb"                => "CzechGangBang",
-            "cgfs"                => "CzechGFS",
-            "czharem"             => "CzechHarem",
-            "czm"                 => "CzechMassage",
-            "czo"                 => "CzechOrgasm",
-            "czps"                => "CzechPawnShop",
-            "css"                 => "CzechStreets",
-            "cztaxi"              => "CzechTaxi",
-            "czt"                 => "CzechTwins",
-            "dts"                 => "DeepThroatSirens",
-            "doan"                => "DiaryOfANanny",
-            "ds"                  => "DungeonSex",
-            "ffr"                 => "FacialsForever",
-            "ff"                  => "FilthyFamily",
-            "fbbg"                => "FirstBGG",
-            "fs"                  => "FuckStudies",
-            "tfcp"                => "FullyClothedPissing",
-            "gdp"                 => "GirlsDoPorn",
-            "Harmony"             => "HarmonyVision",
-            "hletee"              => "HelplessTeens",
-            "jlmf"                => "JessieLoadsMonsterFacials",
-            "lang"                => "LANewGirl",
-            "mmp"                 => "MMPNetwork",
-            "mbc"                 => "MyBabysittersClub",
-            "nvg"                 => "NetVideoGirls",
-            "oo"                  => "Only-Opaques",
-            "os"                  => "Only-Secretaries",
-            "oss"                 => "OnlySilAndSatin",
-            "psus"                => "PascalsSubSluts",
-            "psp"                 => "PorsntarsPunishment",
-            "pdmqfo"              => "QuestForOrgasm",
-            "sed"                 => "SexualDisgrace",
-            "sislov"              => "SisLovesMe",
-            "tslw"                => "SlimeWave",
-            "stre"                => "StrictRestraint",
-            "t18"                 => "Taboo18",
-            "tsma"                => "TeenSexMania",
-            "tsm"                 => "TeenSexMovs",
-            "ttw"                 => "TeensInTheWoods",
-            "tgw"                 => "ThaiGirlsWild",
-            "taob"                => "TheArtOfBlowJob",
-            "trwo"                => "TheRealWorkout",
-            "tt"                  => "TryTeens",
-            "vp"                  => "VIPissy",
-            "wrh"                 => "WeAreHairy",
-            "yt"                  => "YoungThroats",
-    ];
-*/

seeds/06_affiliates.js

@@ -208,6 +208,11 @@ const affiliates = [
 		url: 'https://www.g2buddy.com/disruptivefilms/go.php?pr=9&su=2&si=119&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
+	{
+		channel: 'sodomysquad',
+		url: 'https://www.g2buddy.com/sodomysquad/go.php?pr=9&su=2&si=137&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
 	// gamma > ags max > next door studios
 	// excluded affiliate links that link back to main site and don't seem to track properly
 	{
@@ -494,6 +499,92 @@ const affiliates = [
 			scene: false, // redirects to Adult Time
 		},
 	},
+	// gamma > vivid
+	{
+		network: 'vivid',
+		url: 'https://www.g2fame.com/vivid/go.php?pr=8&su=2&si=330&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+		parameters: {
+			scene: false, // redirects to homepage
+		},
+	},
+	{
+		channel: 'wheretheboysarent',
+		url: 'https://www.g2fame.com/wheretheboysarent/go.php?pr=8&su=2&si=368&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'thebrats',
+		url: 'https://www.g2fame.com/thebrats/go.php?pr=8&su=2&si=369&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	// gamma > zero tolerance
+	{
+		network: 'zerotolerance',
+		url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'zerotolerancefilms',
+		url: 'https://www.g2fame.com/zerotolerancefilms/go.php?pr=8&su=2&si=507&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: '3rddegreefilms',
+		url: 'https://www.g2fame.com/3rddegreefilms/go.php?pr=8&su=2&si=537&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'addicted2girls',
+		url: 'https://www.g2fame.com/addicted2girls/go.php?pr=8&su=2&si=477&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'genderxfilms',
+		url: 'https://www.g2fame.com/genderxfilms/go.php?pr=8&su=2&si=397&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'gangbangcreampie',
+		url: 'https://www.g2fame.com/gangbangcreampie/go.php?pr=8&su=2&si=656&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'gloryholesecrets',
+		url: 'https://www.g2fame.com/gloryholesecrets/go.php?pr=8&su=2&si=655&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'tabooheat',
+		url: 'https://www.g2fame.com/tabooheat/go.php?pr=8&su=2&si=552&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'wicked',
+		url: 'https://www.g2fame.com/wicked/go.php?pr=8&su=2&si=371&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	// gamma > independent channels
+	{
+		channel: 'biphoria',
+		url: 'https://www.g2fame.com/biphoria/go.php?pr=8&su=2&si=418&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'burningangel',
+		url: 'https://www.g2fame.com/burningangel/go.php?pr=8&su=2&si=174&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'chaosmen',
+		url: 'https://www.g2fame.com/chaosmen/go.php?pr=8&su=2&si=608&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'diabolic',
+		url: 'https://www.g2fame.com/diabolic/go.php?pr=8&su=2&si=523&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
 	// kelly madison / 8k
 	{
 		network: 'kellymadison',
@@ -628,6 +719,11 @@ const affiliates = [
 		url: 'https://register.join-toughlovex.com/track/MzAwMDA5NzkuMy43Ni4xOTcuMC4wLjAuMC4w',
 		comment: 'rev share',
 	},
+	{
+		channel: 'hardwerk',
+		url: 'https://register.hardwerk.com/track/MzAwMDA5NzkuMy4xNTEuMzM5LjAuMC4wLjAuMA',
+		comment: 'rev share',
+	},
 	// radical > topwebmodels
 	{
 		network: 'topwebmodels',
@@ -854,6 +950,46 @@ const affiliates = [
 			query: 'ref=4c331ef6',
 		},
 	},
+	// POV Porn Cash / HussiePass
+	{
+		network: 'hussiepass',
+		url: 'https://secure.hussiepass.com/track/MTk0NS4xLjUuNy4wLjAuMC4wLjA',
+		comment: '50% revshare',
+		parameters: {
+			// hussiepass website does not show network scenes
+			channelScenes: false,
+		},
+	},
+	{
+		channel: 'povpornstars',
+		url: 'https://join.povpornstars.com/track/MTk0NS4xLjMuNS4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'interracialpovs',
+		url: 'https://join.interracialpovs.com/track/MTk0NS4xLjYuOC4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'ravebunnys',
+		url: 'https://secure.ravebunnys.com/track/MTk0NS4xLjExLjI5LjAuMC4wLjAuMA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'hotandtatted',
+		url: 'https://join.hotandtatted.com/track/MTk0NS4xLjEwLjEyLjAuMC4wLjAuMA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'seehimfuck',
+		url: 'https://join.seehimfuck.com/track/MTk0NS4xLjcuOS4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'seehimsolo',
+		url: 'https://join.seehimsolo.com/track/MTk0NS4xLjguMTAuMC4wLjAuMC4w',
+		comment: '50% revshare',
+	},
 	// etc
 	{
 		network: 'bang',

src/actors.js

@@ -351,6 +351,7 @@ function curateProfileEntry(profile) {
 		tattoos: profile.tattoos,
 		blood_type: profile.bloodType,
 		avatar_media_id: profile.avatarMediaId || null,
+		updated_at: knex.raw('DEFAULT'), // default should be NOW(), this will update the column
 	};
 
 	return curatedProfileEntry;

src/knex.js

@@ -3,7 +3,7 @@
 const config = require('config');
 const knex = require('knex');
 
-module.exports = knex({
+const knexInstance = knex({
 	client: 'pg',
 	connection: config.database.owner,
 	pool: config.database.pool,
@@ -11,3 +11,23 @@ module.exports = knex({
 	asyncStackTraces: process.env.NODE_ENV === 'development',
 	// debug: process.env.NODE_ENV === 'development',
 });
+
+knexInstance.on('query', function onQuery(query) {
+	const bindingCount = query.bindings?.length ?? 0;
+
+	if (bindingCount > 50000) {
+		const error = new Error(`[knex] Dangerous query: ${bindingCount} bindings detected: ${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`);
+
+		Error.captureStackTrace(error, onQuery);
+		// console.error(error);
+
+		throw error; // optionally hard-fail so you get a real stack trace
+	}
+});
+
+knexInstance.on('query-error', (error, query) => {
+	error.knexSql = `${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`;
+	error.knexBindingCount = query.bindings?.length;
+});
+
+module.exports = knexInstance;

src/media.js

@@ -23,7 +23,7 @@ const logger = require('./logger')(__filename);
 const argv = require('./argv');
 const knex = require('./knex');
 const http = require('./utils/http');
-const bulkInsert = require('./utils/bulk-insert');
+const batchInsert = require('./utils/batch-insert');
 const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 const { fetchEntityReleaseIds } = require('./entity-releases');
@@ -647,6 +647,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
 	const res = await http.get(source.src, {
 		limits: 'media',
 		headers: {
+			host: new URL(source.src).hostname,
 			...(source.referer && { referer: source.referer }),
 			...(source.host && { host: source.host }),
 		},
@@ -923,7 +924,7 @@ async function storeMedias(baseMedias, options) {
 	const newMediaEntries = newMediaWithEntries.filter((media) => media.newEntry).map((media) => media.entry);
 
 	try {
-		await bulkInsert('media', newMediaEntries, false);
+		await batchInsert('media', newMediaEntries, { confict: false });
 
 		return [...newMediaWithEntries, ...existingHashMedias];
 	} catch (error) {
@@ -992,11 +993,11 @@ async function associateReleaseMedia(releases, type = 'release') {
 				.filter(Boolean);
 
 			if (associations.length > 0) {
-				await bulkInsert(`${type}s_${role}`, associations, false);
+				await batchInsert(`${type}s_${role}`, associations, { conflict: false });
 			}
 		} catch (error) {
 			if (error.entries) {
-				logger.error(util.inspect(error.entries, null, null, { color: true }));
+				logger.error(util.inspect(error.entries.slice(0, 2), null, null, { color: true }), `${Math.min(error.entries.length, 2)} of ${error.length}`);
 			}
 
 			logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'})`);

src/scrapers/actors.js

@@ -5,7 +5,7 @@ const angelogodshackoriginal = require('./angelogodshackoriginal');
 // const americanpornstar = require('./americanpornstar'); // offline
 const aziani = require('./aziani');
 const badoink = require('./badoink');
-// const bamvisions = require('./bamvisions');
+const bamvisions = require('./bamvisions');
 const bang = require('./bang');
 const bradmontana = require('./bradmontana');
 const cherrypimps = require('./cherrypimps');
@@ -140,6 +140,7 @@ module.exports = {
 	purgatoryx: radical,
 	topwebmodels: radical,
 	lucidflix: radical,
+	hardwerk: radical,
 	// hush / hussiepass
 	eyeontheguy: hush,
 	hushpass: hush,
@@ -215,7 +216,7 @@ module.exports = {
 	angelogodshackoriginal,
 	babevr: badoink,
 	badoinkvr: badoink,
-	// bamvisions, // DNS error, site offline?
+	bamvisions,
 	bang,
 	meidenvanholland: bluedonkeymedia, // Vurig Vlaanderen uses same database
 	boobpedia,

src/scrapers/aylo.js

@@ -57,7 +57,7 @@ function getCovers(images, target = 'cover') {
 }
 
 function getVideos(data) {
-	const teaserSources = data.videos.mediabook?.files;
+	const teaserSources = data.videos?.mediabook?.files;
 	const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
 
 	const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
@@ -84,7 +84,7 @@ function scrapeLatestX(data, site, filterChannel, options) {
 
 	release.url = `${basepath}/${data.id}/${slugify(release.title)}`; // spartanId doesn't work in URLs
 	release.date = new Date(data.dateReleased);
-	release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
+	release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
 
 	release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
 	release.tags = data.tags.map((tag) => tag.name);
@@ -127,6 +127,10 @@ async function scrapeLatest(items, site, filterChannel, options) {
 }
 
 function scrapeRelease(data, url, channel, networkName, options) {
+	if (Array.isArray(data)) {
+		return null;
+	}
+
 	const release = {};
 
 	const { title, description } = data;
@@ -136,7 +140,7 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	release.description = description;
 
 	release.date = new Date(data.dateReleased);
-	release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
+	release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
 
 	release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
 	release.tags = data.tags.map((tag) => tag.name);
@@ -144,7 +148,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	[release.poster, ...release.photos] = getThumbs(data).map((src) => ({
 		src,
 		referer: url,
-		host: 'mediavault-private-fl.project1content.com',
 	}));
 
 	const { teaser, trailer } = getVideos(data);
@@ -270,7 +273,7 @@ async function fetchLatest(site, page = 1, options) {
 		return null;
 	}
 
-	const { instanceToken } = options.beforeNetwork?.instanceToken
+	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
 		? options.beforeNetwork
 		: await getSession(site, options.parameters, url);
 

src/scrapers/gamma.js

@@ -21,6 +21,10 @@ async function scrapeApiReleases(json, site, options) {
 			return acc;
 		}
 
+		if (options.parameters?.filterChannel && scene.mainChannel?.id !== options.parameters.filterChannel) {
+			return acc;
+		}
+
 		const release = {
 			entryId: scene.clip_id,
 			description: scene.description,
@@ -128,7 +132,7 @@ async function fetchApiCredentials(referer, site) {
 }
 
 async function fetchLatest(site, page = 1, options, _preData, upcoming = false) {
-	const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
+	const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
 	const { apiUrl } = await fetchApiCredentials(referer, site);
 	const slug = options.parameters.querySlug || site.slug;
 
@@ -179,7 +183,7 @@ async function getFullPhotos(entryId, site, parameters) {
 }
 
 async function getThumbs(entryId, site, parameters) {
-	const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
+	const referer = parameters?.referer || `${parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
 	const { apiUrl } = await fetchApiCredentials(referer, site);
 
 	const res = await unprint.post(apiUrl, {
@@ -237,7 +241,7 @@ async function scrapeScene(data, site, options, movieScenes) {
 	release.entryId = data.clip_id || data.movie_id;
 	release.title = curateTitle(data.title, site);
 	release.duration = data.length;
-	release.date = (data.date && new Date(data.date * 1000)) || unprint.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
+	release.date = (data.date && new Date(data.date * 1000)) || unprint.extractDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
 	release.director = data.directors[0]?.name || null;
 
 	release.actors = data.actors.map((actor) => ({
@@ -298,7 +302,7 @@ async function scrapeScene(data, site, options, movieScenes) {
 }
 
 async function fetchScene(url, site, baseRelease, options) {
-	const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
+	const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.origin : site.origin}/en/videos`;
 	const { apiUrl } = await fetchApiCredentials(referer, site);
 
 	const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1];

src/scrapers/nubiles.js

@@ -70,8 +70,7 @@ function scrapeAll(scenes, entity) {
 async function fetchLatest(site, page = 1) {
 	const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
 
-	const res = await unprint.get(url, {
-		interface: 'request',
+	const res = await unprint.browser(url, {
 		selectAll: '.content-grid-item',
 	});
 
@@ -86,9 +85,8 @@ async function fetchUpcoming(site) {
 	if (site.parameters?.upcoming) {
 		const url = `${site.url}/video/upcoming`;
 
-		const res = await unprint.get(url, {
+		const res = await unprint.browser(url, {
 			selectAll: '.content-grid-item',
-			interface: 'request',
 		});
 
 		if (res.ok) {
@@ -139,9 +137,7 @@ async function scrapeScene({ query }, { url, entity, include }) {
 }
 
 async function fetchScene(url, entity, _baseRelease, include) {
-	const res = await unprint.get(url, {
-		interface: 'request',
-	});
+	const res = await unprint.browser(url);
 
 	if (res.ok) {
 		return scrapeScene(res.context, { url, entity, include });
@@ -185,9 +181,7 @@ async function findModel(actor, entity) {
 
 	const url = `${origin}/model/alpha/${firstLetter}`;
 
-	const resModels = await unprint.get(url, {
-		interface: 'request',
-	});
+	const resModels = await unprint.browser(url);
 
 	if (!resModels.ok) {
 		return resModels.status;
@@ -217,9 +211,7 @@ async function fetchProfile(actor, { entity }) {
 	const model = await findModel(actor, entity);
 
 	if (model) {
-		const resModel = await unprint.get(model.url, {
-			interface: 'request',
-		});
+		const resModel = await unprint.browser(model.url);
 
 		if (resModel.ok) {
 			return scrapeProfile(resModel.context, model.avatar);

src/scrapers/radical.js

@@ -215,7 +215,7 @@ function scrapeProfile(data, channel, scenes, parameters) {
 
 async function fetchProfile(actor, { channel, parameters }) {
 	const endpoint = await fetchEndpoint(channel);
-	const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
+	const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
 
 	if (res.ok && res.body.pageProps?.model) {
 		return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);

src/scrapers/vixen.js

@@ -502,6 +502,7 @@ async function fetchScene(url, channel, baseRelease, options) {
 	}
 
 	const res = await unprint.get(url, {
+		interface: 'request',
 		useBrowser: !!options.parameters?.useBrowser,
 	});
 

src/tags.js

@@ -4,6 +4,7 @@ const logger = require('./logger')(__filename);
 const knex = require('./knex');
 
 const { fetchEntityReleaseIds } = require('./entities');
+const { updateSceneSearch } = require('./update-search');
 
 const slugify = require('./utils/slugify');
 const batchInsert = require('./utils/batch-insert');
@@ -199,6 +200,8 @@ async function reassociateTagEntries(tagEntries, rematch) {
 		tag_id: matchedTags[slugify(tagEntry.original_tag)],
 	})).filter((tagEntry) => tagEntry.tag_id);
 
+	const sceneIds = Array.from(new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id))).filter(Boolean);
+
 	if (updatedTagEntries.length > 0) {
 		const trx = await knex.transaction();
 
@@ -212,10 +215,13 @@ async function reassociateTagEntries(tagEntries, rematch) {
 		})), {
 			conflict: false,
 			transaction: trx,
+			commit: true,
 		});
+
+		await updateSceneSearch(sceneIds);
 	}
 
-	logger.info(`Updated ${updatedTagEntries.length} tags in ${new Set(updatedTagEntries.map((tagEntry) => tagEntry.release_id)).size} scenes`);
+	logger.info(`Updated ${updatedTagEntries.length} tags in ${sceneIds.length} scenes`);
 }
 
 async function reassociateReleaseTags(rawSceneIds, rematch) {

src/tools/gamma_banners.js

@@ -0,0 +1,136 @@
+'use strict';
+
+const unprint = require('unprint');
+const fs = require('fs');
+const { Readable } = require('stream');
+const { pipeline } = require('stream/promises');
+
+const knex = require('../knex');
+const argv = require('../argv');
+const slugify = require('../utils/slugify');
+
+const apiUrl = 'https://vjoc5ygk89-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(3.33.0)%3B%20Browser%20(lite)%3B%20react%20(16.8.6)%3B%20react-instantsearch%20(5.7.0)%3B%20JS%20Helper%20(2.28.1)&x-algolia-application-id=VJOC5YGK89&x-algolia-api-key=c5546bdfb4d3f31daf49ed3bb1463561';
+
+async function fetchBanners() {
+	const res = await unprint.post(
+		apiUrl,
+		{
+			requests: [
+				{
+					indexName: 'creatives',
+					params: new URLSearchParams({
+						hitsPerPage: 1000,
+						maxValuesPerFacet: 100,
+						page: 0,
+						filters: '(ProgramType:Legacy OR ProgramType:Internal) AND NOT OverlayActive:false',
+						facets: '["SceneActors","SceneCategories","ProgramName","Size","Niche","MediaExt","SiteTag","OverlayName"]',
+						facetFilters: `[["SiteTag:${argv.site}"],["MediaExt:jpg", "MediaExt:png", "MediaExt:gif"]]`,
+					}).toString(),
+				},
+			],
+		},
+		{
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				referer: 'https://creatives.gammae.com/',
+			},
+		},
+	);
+
+	if (res.ok && res.data.results[0]) {
+		return res.data.results[0].hits;
+	}
+
+	console.error(`Failed API request (${res.status}): ${res.body}`);
+
+	return null;
+}
+
+async function matchTags(rawTags) {
+	if (!rawTags) {
+		return [];
+	}
+
+	const tags = rawTags
+		.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
+		.filter(Boolean);
+
+	const tagEntries = await knex('tags')
+		.select('tags.slug', 'aliases.slug as alias_slug')
+		.whereIn(knex.raw('lower(tags.name)'), tags)
+		.leftJoin('tags as aliases', 'aliases.id', 'tags.alias_for')
+		.orderByRaw('CASE WHEN tags.alias_for IS NOT NULL THEN aliases.priority ELSE tags.priority END DESC');
+
+	return tagEntries.map((tagEntry) => tagEntry.alias_slug || tagEntry.slug);
+}
+
+async function init() {
+	const banners = await fetchBanners();
+
+	if (!banners) {
+		return;
+	}
+
+	await banners.reduce(async (chain, banner) => {
+		await chain;
+
+		const channel = slugify(banner.SiteTag, '');
+		const url = unprint.prefixUrl(banner.MediaLocation || banner.CreativeURL, 'https://cdn.banhq.com');
+
+		if (!url) {
+			console.log('No URL found');
+			console.log(banner);
+			return;
+		}
+
+		const tags = await matchTags([
+			...banner.Tags?.map((tag) => tag.Value) || [],
+			...banner.SceneCategories || [],
+			banner.Niche,
+		].filter(Boolean));
+
+		const fileTags = tags.slice(0, 4).join('_');
+		const fileActors = banner.SceneActors?.slice(0, 2).map((actor) => slugify(actor, '_')).join('_');
+
+		// tags are unreliable and describe entire scene, not banner, don't include by default
+		const segments = [channel, banner.Width, banner.Height, banner.MediaID, argv.actors?.[0] !== false && fileActors].filter(Boolean);
+		const filename = `${segments.join('_')}${argv.tags && argv.tags ? `-${fileTags}` : ''}.${banner.MediaExt || 'jpg'}`;
+
+		const filepath = `/tmp/gamma/${channel}/${filename}`;
+
+		if (argv.inspect) {
+			console.log(banner);
+		}
+
+		if (argv.preview) {
+			console.log(`Preview ${url}: ${filepath}`);
+			return;
+		}
+
+		await fs.promises.mkdir(`/tmp/gamma/${channel}`, { recursive: true });
+
+		try {
+			const res = await fetch(url);
+
+			if (res.ok && res.body) {
+				const writer = fs.createWriteStream(filepath);
+
+				await pipeline(Readable.fromWeb(res.body), writer);
+
+				if (argv.actors) {
+					console.log(`Saved ${url} to ${filepath}`);
+				} else {
+					console.log(`Saved ${url} to ${filepath}, actors ${banner.SceneActors?.join(', ') || ''}`);
+				}
+			} else {
+				console.log(`Failed to fetch ${url} (${res.status})`);
+			}
+		} catch (error) {
+			console.log(`Failed to fetch ${url}: ${error.message}`);
+		}
+	}, Promise.resolve());
+
+	await knex.destroy();
+}
+
+init();

src/tools/huge-query.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const knex = require('../knex');
+
+async function init() {
+	const data = Array.from({ length: 100_000 }, (value, index) => ({
+		id: `test_affiliate_${index}`,
+	}));
+
+	await knex('affiliates').insert(data);
+
+	console.log('Done!');
+}
+
+init();

src/tools/manticore-scenes.js

@@ -41,7 +41,7 @@ async function fetchScenes() {
 			studios.name as studio_name,
 			grandparents.id as parent_network_id,
 			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
+			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
 			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
 			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
 			COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
@@ -136,6 +136,14 @@ async function init() {
 			 dupe_index int
 		)`);
 
+		await utilsApi.sql('drop table if exists scenes_tags');
+		await utilsApi.sql(`create table scenes_tags (
+			 id int,
+			 scene_id int,
+			 tag_id int,
+			 actor_id int
+		)`);
+
 		console.log('Recreated scenes table');
 		console.log('Fetching scenes from primary database');
 
@@ -143,49 +151,62 @@ async function init() {
 
 		console.log('Fetched scenes from primary database');
 
-		const docs = scenes.map((scene) => {
+		const docs = scenes.flatMap((scene) => {
 			const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
 			const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
 			const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
 
-			return {
-				replace: {
-					index: 'scenes',
-					id: scene.id,
-					doc: {
-						title: scene.title || undefined,
-						title_filtered: filteredTitle || undefined,
-						date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-						created_at: Math.round(scene.created_at.getTime() / 1000),
-						effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-						is_showcased: scene.showcased,
-						entry_id: scene.entry_id || undefined,
-						shoot_id: scene.shoot_id || undefined,
-						channel_id: scene.channel_id,
-						channel_slug: scene.channel_slug,
-						channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
-						network_id: scene.network_id || undefined,
-						network_slug: scene.network_slug || undefined,
-						network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
-						studio_id: scene.studio_id || undefined,
-						studio_slug: scene.studio_slug || undefined,
-						studio_name: scene.studio_name || undefined,
-						entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-						actor_ids: scene.actors.map((actor) => actor.f1),
-						actors: scene.actors.map((actor) => actor.f2).join(),
-						tag_ids: scene.tags.map((tag) => tag.f1),
-						tags: flatTags.join(' '),
-						movie_ids: scene.movies.map((movie) => movie.f1),
-						movies: scene.movies.map((movie) => movie.f2).join(' '),
-						serie_ids: scene.series.map((serie) => serie.f1),
-						series: scene.series.map((serie) => serie.f2).join(' '),
-						fingerprints: scene.fingerprints.join(' '),
-						meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
-						stashed: scene.stashed || 0,
-						dupe_index: scene.dupe_index || 0,
+			return [
+				{
+					replace: {
+						index: 'scenes',
+						id: scene.id,
+						doc: {
+							title: scene.title || undefined,
+							title_filtered: filteredTitle || undefined,
+							date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
+							created_at: Math.round(scene.created_at.getTime() / 1000),
+							effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
+							is_showcased: scene.showcased,
+							entry_id: scene.entry_id || undefined,
+							shoot_id: scene.shoot_id || undefined,
+							channel_id: scene.channel_id,
+							channel_slug: scene.channel_slug,
+							channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
+							network_id: scene.network_id || undefined,
+							network_slug: scene.network_slug || undefined,
+							network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
+							studio_id: scene.studio_id || undefined,
+							studio_slug: scene.studio_slug || undefined,
+							studio_name: scene.studio_name || undefined,
+							entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
+							actor_ids: scene.actors.map((actor) => actor.f1),
+							actors: scene.actors.map((actor) => actor.f2).join(),
+							tag_ids: scene.tags.map((tag) => tag.f1),
+							tags: flatTags.join(' '),
+							movie_ids: scene.movies.map((movie) => movie.f1),
+							movies: scene.movies.map((movie) => movie.f2).join(' '),
+							serie_ids: scene.series.map((serie) => serie.f1),
+							series: scene.series.map((serie) => serie.f2).join(' '),
+							fingerprints: scene.fingerprints.join(' '),
+							meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
+							stashed: scene.stashed || 0,
+							dupe_index: scene.dupe_index || 0,
+						},
 					},
 				},
-			};
+				...scene.tags.map((tag) => ({
+					replace: {
+						index: 'scenes_tags',
+						// id: scene.id,
+						doc: {
+							scene_id: scene.id,
+							tag_id: tag.f1,
+							actor_id: tag.f5,
+						},
+					},
+				})),
+			];
 		});
 
 		// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {

src/tools/manticore-stashes.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const config = require('config');
+const manticore = require('manticoresearch');
+
+const knex = require('../knex');
+const chunk = require('../utils/chunk');
+
+const mantiClient = new manticore.ApiClient();
+
+mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
+
+const utilsApi = new manticore.UtilsApi(mantiClient);
+const indexApi = new manticore.IndexApi(mantiClient);
+
+async function syncStashes(domain = 'scene') {
+	await utilsApi.sql(`truncate table ${domain}s_stashed`);
+
+	const stashes = await knex(`stashes_${domain}s`)
+		.select(
+			`stashes_${domain}s.id as stashed_id`,
+			`stashes_${domain}s.${domain}_id`,
+			'stashes.id as stash_id',
+			'stashes.user_id as user_id',
+			`stashes_${domain}s.created_at as created_at`,
+		)
+		.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
+
+	await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
+		await chain;
+
+		const stashDocs = stashChunk.map((stash) => ({
+			replace: {
+				index: `${domain}s_stashed`,
+				id: stash.stashed_id,
+				doc: {
+					[`${domain}_id`]: stash[`${domain}_id`],
+					stash_id: stash.stash_id,
+					user_id: stash.user_id,
+					created_at: Math.round(stash.created_at.getTime() / 1000),
+				},
+			},
+		}));
+
+		await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
+
+		console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
+	}, Promise.resolve());
+}
+
+async function init() {
+	await utilsApi.sql('drop table if exists scenes_stashed');
+
+	await utilsApi.sql(`create table if not exists scenes_stashed (
+		scene_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await utilsApi.sql('drop table if exists movies_stashed');
+
+	await utilsApi.sql(`create table if not exists movies_stashed (
+		movie_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await utilsApi.sql('drop table if exists actors_stashed');
+
+	await utilsApi.sql(`create table if not exists actors_stashed (
+		actor_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await syncStashes('scene');
+	await syncStashes('actor');
+	await syncStashes('movie');
+
+	console.log('Done!');
+
+	knex.destroy();
+}
+
+init();

src/update-search.js

@@ -16,14 +16,14 @@ async function updateManticoreStashedScenes(docs) {
 	await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
 		await chain;
 
-		const sceneIds = docsChunk.map((doc) => doc.replace.id);
+		const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
 
 		const stashes = await knex('stashes_scenes')
 			.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
 			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
 			.whereIn('scene_id', sceneIds);
 
-		const stashDocs = docsChunk.flatMap((doc) => {
+		const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
 			const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
 
 			if (sceneStashes.length === 0) {
@@ -50,6 +50,25 @@ async function updateManticoreStashedScenes(docs) {
 		if (stashDocs.length > 0) {
 			await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
 		}
+
+		const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
+
+		if (deleteSceneIds.length > 0) {
+			await indexApi.callDelete({
+				index: 'scenes_stashed',
+				query: {
+					bool: {
+						must: [
+							{
+								in: {
+									scene_id: deleteSceneIds,
+								},
+							},
+						],
+					},
+				},
+			});
+		}
 	}, Promise.resolve());
 }
 
@@ -128,9 +147,20 @@ async function updateManticoreSceneSearch(releaseIds) {
 			studios.showcased
     `, releaseIds && [releaseIds]);
 
-	// console.log(scenes.rows);
+	const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
+
+	const docs = releaseIds.map((sceneId) => {
+		const scene = scenesById[sceneId];
+
+		if (!scene) {
+			return {
+				delete: {
+					index: 'scenes',
+					id: sceneId,
+				},
+			};
+		}
 
-	const docs = scenes.rows.map((scene) => {
 		const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
 		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
 		const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
@@ -291,7 +321,20 @@ async function updateManticoreMovieSearch(movieIds) {
 			movies_covers.*
     `, movieIds && [movieIds]);
 
-	const docs = movies.rows.map((movie) => {
+	const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
+
+	const docs = movieIds.map((movieId) => {
+		const movie = moviesById[movieId];
+
+		if (!movie) {
+			return {
+				delete: {
+					index: 'movies',
+					id: movieId,
+				},
+			};
+		}
+
 		const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
 			id: tag.f1,
 			name: tag.f2,

src/utils/batch-insert.js

@@ -4,11 +4,12 @@ const knex = require('../knex');
 const chunk = require('./chunk');
 const logger = require('../logger')(__filename);
 
+const chunkTarget = 50_000; // PostgreSQL allows 65,535 binding parameters, allow for a bit of margin
+
 // improved version of bulkInsert
 async function batchInsert(table, items, {
 	conflict = true,
 	update = false,
-	chunkSize = 1000,
 	concurrent = false,
 	transaction,
 	commit = false,
@@ -17,6 +18,10 @@ async function batchInsert(table, items, {
 		throw new Error('No table specified for batch insert');
 	}
 
+	if (conflict && update) {
+		throw new Error('Batch insert conflict must specify columns, or update must be disabled');
+	}
+
 	if (!Array.isArray(items)) {
 		throw new Error('Batch insert items are not an array');
 	}
@@ -25,8 +30,20 @@ async function batchInsert(table, items, {
 		return [];
 	}
 
-	const chunks = chunk(items, chunkSize);
+	// PostgreSQL's bindings limit applies to individual values, so item size needs to be taken into account
+	const itemSize = items.reduce((acc, item) => Math.max(acc, Object.keys(item).length), 0);
+
+	if (itemSize === 0) {
+		throw new Error('Batch insert items are empty');
+	}
+
+	const chunks = chunk(items, Math.floor(chunkTarget / itemSize));
 	const conflicts = [].concat(conflict).filter((column) => typeof column === 'string'); // conflict might be 'true'
+
+	if (conflicts.length > 0 && !update) {
+		throw new Error('Batch insert conflict columns must be specified together with update');
+	}
+
 	const trx = transaction || await knex.transaction();
 
 	try {
@@ -49,12 +66,6 @@ async function batchInsert(table, items, {
 						.onConflict(conflicts)
 						.merge();
 				}
-
-				throw new Error('Batch insert conflict columns must be specified together with update');
-			}
-
-			if (conflict && update) {
-				throw new Error('Batch insert conflict must specify columns, or update must be disabled');
 			}
 
 			// error on any conflict

tests/profiles.js

@@ -145,7 +145,7 @@ const actors = [
 	// perv city
 	{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
 	{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
-	// { entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] }, // site offline as of 2026-02-25
+	{ entity: 'bamvisions', name: 'Abella Danger', fields: ['avatar', 'height', 'measurements'] }, // site offline as of 2026-02-25
 	// radical
 	{ entity: 'bjraw', name: 'Nikki Knightly', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
 	{ entity: 'gotfilled', name: 'Alexa Chains', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
@@ -153,6 +153,7 @@ const actors = [
 	{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
 	{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
 	{ entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] },
+	{ entity: 'hardwerk', name: 'Luna Silver', fields: ['avatar', 'gender'] },
 	// wankz
 	{ entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] },
 	{ entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] },