1.250.38

.eslintrc

@@ -27,7 +27,7 @@
 		"require-await": "off",
         "no-param-reassign": ["error", {
             "props": true,
-            "ignorePropertyModificationsFor": ["state", "acc", "req"]
+            "ignorePropertyModificationsFor": ["state", "acc", "req", "error"]
         }]
     },
     "globals": {

config/default.js

@@ -188,6 +188,8 @@ module.exports = {
 			'wishescumtrue',
 			// hentaied
 			'somegore',
+			// digital playground
+			'digitalplayground', // no longer updates, produces a bunch of garbage for some reason
 		],
 		networks: [
 			// dummy network for testing
@@ -279,7 +281,7 @@ module.exports = {
 		trailerQuality: [540, 720, 960, 480, 1080, 360, 320, 1440, 1600, 1920, 2160, 270, 240, 180],
 		limit: 25, // max number of photos per release
 		attempts: 2,
-		flushOrphaned: true,
+		flushOrphaned: false,
 		flushWindow: 1000,
 		streams: {
 			enabled: true, // fetch streams

migrations/20260302222545_series_alt_descriptions.js

@@ -0,0 +1,13 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('series', (table) => {
+		table.specificType('alt_descriptions', 'text ARRAY');
+		table.json('attributes');
+	});
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('series', (table) => {
+		table.dropColumn('alt_descriptions');
+		table.dropColumn('attributes');
+	});
+};

migrations/20260304020542_scene_actor_tags.js

@@ -0,0 +1,21 @@
+exports.up = async function(knex) {
+	await knex.schema.alterTable('releases_tags', (table) => {
+		table.integer('actor_id')
+			.references('id')
+			.inTable('actors');
+
+		table.dropUnique(['tag_id', 'release_id']);
+	});
+
+	await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
+};
+
+exports.down = async function(knex) {
+	await knex.schema.alterTable('releases_tags', (table) => {
+		table.dropColumn('actor_id');
+
+		table.unique(['tag_id', 'release_id']);
+	});
+
+	await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
+};

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.250.15",
+    "version": "1.250.38",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.250.15",
+            "version": "1.250.38",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.250.15",
+    "version": "1.250.38",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/00_tags.js

@@ -3024,6 +3024,8 @@ const priorities = [ // higher index is higher priority
 	['blowbang', 'orgy'],
 	['gangbang'],
 	['gay', 'transsexual', 'bisexual', 'hentai'],
+	['pissing'],
+	['compilation', 'bts'],
 ].reduce((acc, slugs, index) => {
 	slugs.forEach((slug) => { acc[slug] = index; });
 

seeds/02_sites.js

@@ -609,6 +609,9 @@ const sites = [
 		slug: 'asgmaxoriginals',
 		url: 'https://www.asgmax.com/en/channel/asgmaxoriginals',
 		parent: 'asgmax',
+		parameters: {
+			queryChannel: 'asgmaxoriginals',
+		},
 	},
 	{
 		name: 'ASG Max Films',
@@ -616,6 +619,9 @@ const sites = [
 		url: 'https://www.asgmax.com/en/channel/asgmaxfilms',
 		parent: 'asgmax',
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmaxfilms',
+		},
 	},
 	{
 		name: 'ASG International',
@@ -623,6 +629,9 @@ const sites = [
 		url: 'https://www.asgmax.com/en/channel/asginternational',
 		parent: 'asgmax',
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asginternational',
+		},
 	},
 	{
 		name: 'ASG Massage',
@@ -631,6 +640,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['massage'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmassage',
+		},
 	},
 	{
 		name: 'ASG Auditions',
@@ -639,6 +651,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['audition'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgauditions',
+		},
 	},
 	{
 		name: 'ASG Free Use',
@@ -647,6 +662,9 @@ const sites = [
 		parent: 'asgmax',
 		tags: ['free-use'],
 		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgfreeuse',
+		},
 	},
 	{
 		name: 'Exeter Hill College',
@@ -655,6 +673,9 @@ const sites = [
 		parent: 'asgmax',
 		hasLogo: false,
 		tags: ['animated'],
+		parameters: {
+			queryChannel: 'asgexeterhillcollege',
+		},
 	},
 	// ASG MAX INDEPENDENT
 	{
@@ -1018,12 +1039,12 @@ const sites = [
 	{
 		name: 'Disruptive Films',
 		slug: 'disruptivefilms',
-		delete: true,
 		url: 'https://www.disruptivefilms.com',
 		parent: 'disruptivefilms',
 		tags: ['gay'],
 		parameters: {
 			queryChannel: 'asgmaxdisruptivefilms',
+			filterChannel: 'asgmaxdisruptivefilms',
 		},
 	},
 	{
@@ -1070,6 +1091,17 @@ const sites = [
 			queryChannel: 'asgmaxtruemale',
 		},
 	},
+	{
+		name: 'Deep Inside',
+		slug: 'deepinside',
+		url: 'https://www.asgmax.com/en/channel/asgmaxdeepinside',
+		parent: 'disruptivefilms',
+		tags: ['gay'],
+		hasLogo: false,
+		parameters: {
+			queryChannel: 'asgmaxdeepinside',
+		},
+	},
 	// AMATEUR ALLURE
 	{
 		name: 'Amateur Allure',
@@ -5422,6 +5454,12 @@ const sites = [
 			referer: 'https://www.girlsway.com',
 		},
 	},
+	{
+		slug: 'lesbianfactor',
+		name: 'Lesbian Factor',
+		url: 'https://www.lesbianfactor.com',
+		parent: 'girlsway',
+	},
 	// HITZEFREI
 	{
 		slug: 'unleashed',
@@ -5532,6 +5570,16 @@ const sites = [
 			latest: 'https://seehimfuck.com',
 		},
 	},
+	{
+		slug: 'seehimsolo',
+		name: 'See Him Solo',
+		url: 'https://seehimsolo.com',
+		tags: ['male-focus', 'solo'],
+		parent: 'hussiepass',
+		parameters: {
+			latest: 'https://seehimsolo.com/categories/movies-2/{page}/latest/',
+		},
+	},
 	{
 		slug: 'interracialpovs',
 		name: 'Interracial POVs',
@@ -10511,10 +10559,11 @@ const sites = [
 		name: 'Look At Her Now',
 		url: 'https://www.lookathernow.com',
 		description: 'Look At Her Now brings you best HD reality porn videos every week. Check out these girls before and after they get some rough pounding.',
-		parameters: { native: true },
-		// parameters: { siteId: 300 },
 		slug: 'lookathernow',
 		parent: 'realitykings',
+		parameters: {
+			siteId: 364,
+		},
 	},
 	{
 		name: 'We Live Together',
@@ -15534,7 +15583,7 @@ exports.seed = (knex) => Promise.resolve()
 			url: site.url,
 			parameters: site.parameters || null,
 			options: site.options,
-			parent_id: networksMap[site.parent],
+			parent_id: networksMap[site.parent] || null,
 			priority: site.priority || 0,
 			independent: !!site.independent,
 			visible: site.visible,

seeds/06_affiliates.js

@@ -208,6 +208,11 @@ const affiliates = [
 		url: 'https://www.g2buddy.com/disruptivefilms/go.php?pr=9&su=2&si=119&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
+	{
+		channel: 'sodomysquad',
+		url: 'https://www.g2buddy.com/sodomysquad/go.php?pr=9&su=2&si=137&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
 	// gamma > ags max > next door studios
 	// excluded affiliate links that link back to main site and don't seem to track properly
 	{
@@ -539,6 +544,47 @@ const affiliates = [
 		url: 'https://www.g2fame.com/genderxfilms/go.php?pr=8&su=2&si=397&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
+	{
+		channel: 'gangbangcreampie',
+		url: 'https://www.g2fame.com/gangbangcreampie/go.php?pr=8&su=2&si=656&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'gloryholesecrets',
+		url: 'https://www.g2fame.com/gloryholesecrets/go.php?pr=8&su=2&si=655&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'tabooheat',
+		url: 'https://www.g2fame.com/tabooheat/go.php?pr=8&su=2&si=552&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'wicked',
+		url: 'https://www.g2fame.com/wicked/go.php?pr=8&su=2&si=371&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	// gamma > independent channels
+	{
+		channel: 'biphoria',
+		url: 'https://www.g2fame.com/biphoria/go.php?pr=8&su=2&si=418&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'burningangel',
+		url: 'https://www.g2fame.com/burningangel/go.php?pr=8&su=2&si=174&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'chaosmen',
+		url: 'https://www.g2fame.com/chaosmen/go.php?pr=8&su=2&si=608&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
+		channel: 'diabolic',
+		url: 'https://www.g2fame.com/diabolic/go.php?pr=8&su=2&si=523&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
 	// kelly madison / 8k
 	{
 		network: 'kellymadison',
@@ -899,6 +945,46 @@ const affiliates = [
 			query: 'ref=4c331ef6',
 		},
 	},
+	// POV Porn Cash / HussiePass
+	{
+		network: 'hussiepass',
+		url: 'https://secure.hussiepass.com/track/MTk0NS4xLjUuNy4wLjAuMC4wLjA',
+		comment: '50% revshare',
+		parameters: {
+			// hussiepass website does not show network scenes
+			channelScenes: false,
+		},
+	},
+	{
+		channel: 'povpornstars',
+		url: 'https://join.povpornstars.com/track/MTk0NS4xLjMuNS4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'interracialpovs',
+		url: 'https://join.interracialpovs.com/track/MTk0NS4xLjYuOC4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'ravebunnys',
+		url: 'https://secure.ravebunnys.com/track/MTk0NS4xLjExLjI5LjAuMC4wLjAuMA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'hotandtatted',
+		url: 'https://join.hotandtatted.com/track/MTk0NS4xLjEwLjEyLjAuMC4wLjAuMA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'seehimfuck',
+		url: 'https://join.seehimfuck.com/track/MTk0NS4xLjcuOS4wLjAuMC4wLjA',
+		comment: '50% revshare',
+	},
+	{
+		channel: 'seehimsolo',
+		url: 'https://join.seehimsolo.com/track/MTk0NS4xLjguMTAuMC4wLjAuMC4w',
+		comment: '50% revshare',
+	},
 	// etc
 	{
 		network: 'bang',

src/knex.js

@@ -3,7 +3,7 @@
 const config = require('config');
 const knex = require('knex');
 
-module.exports = knex({
+const knexInstance = knex({
 	client: 'pg',
 	connection: config.database.owner,
 	pool: config.database.pool,
@@ -11,3 +11,23 @@ module.exports = knex({
 	asyncStackTraces: process.env.NODE_ENV === 'development',
 	// debug: process.env.NODE_ENV === 'development',
 });
+
+knexInstance.on('query', function onQuery(query) {
+	const bindingCount = query.bindings?.length ?? 0;
+
+	if (bindingCount > 10000) {
+		const error = new Error(`[knex] Dangerous query: ${bindingCount} bindings detected: ${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`);
+
+		Error.captureStackTrace(error, onQuery);
+		// console.error(error);
+
+		throw error; // optionally hard-fail so you get a real stack trace
+	}
+});
+
+knexInstance.on('query-error', (error, query) => {
+	error.knexSql = `${query.sql?.slice(0, 200)}${query.sql?.length > 200 ? '...' : ''}`;
+	error.knexBindingCount = query.bindings?.length;
+});
+
+module.exports = knexInstance;

src/media.js

@@ -647,6 +647,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
 	const res = await http.get(source.src, {
 		limits: 'media',
 		headers: {
+			host: new URL(source.src).hostname,
 			...(source.referer && { referer: source.referer }),
 			...(source.host && { host: source.host }),
 		},

src/scrapers/aylo.js

@@ -57,7 +57,7 @@ function getCovers(images, target = 'cover') {
 }
 
 function getVideos(data) {
-	const teaserSources = data.videos.mediabook?.files;
+	const teaserSources = data.videos?.mediabook?.files;
 	const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
 
 	const teaser = teaserSources && Object.values(teaserSources).map((source) => ({
@@ -84,7 +84,7 @@ function scrapeLatestX(data, site, filterChannel, options) {
 
 	release.url = `${basepath}/${data.id}/${slugify(release.title)}`; // spartanId doesn't work in URLs
 	release.date = new Date(data.dateReleased);
-	release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
+	release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
 
 	release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
 	release.tags = data.tags.map((tag) => tag.name);
@@ -127,6 +127,10 @@ async function scrapeLatest(items, site, filterChannel, options) {
 }
 
 function scrapeRelease(data, url, channel, networkName, options) {
+	if (Array.isArray(data)) {
+		return null;
+	}
+
 	const release = {};
 
 	const { title, description } = data;
@@ -136,7 +140,7 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	release.description = description;
 
 	release.date = new Date(data.dateReleased);
-	release.duration = data.videos.mediabook?.length > 1 ? data.videos.mediabook.length : null;
+	release.duration = data.videos?.mediabook?.length > 1 ? data.videos.mediabook.length : null;
 
 	release.actors = data.actors.map((actor) => ({ name: actor.name, gender: actor.gender }));
 	release.tags = data.tags.map((tag) => tag.name);
@@ -144,7 +148,6 @@ function scrapeRelease(data, url, channel, networkName, options) {
 	[release.poster, ...release.photos] = getThumbs(data).map((src) => ({
 		src,
 		referer: url,
-		host: 'mediavault-private-fl.project1content.com',
 	}));
 
 	const { teaser, trailer } = getVideos(data);
@@ -270,7 +273,7 @@ async function fetchLatest(site, page = 1, options) {
 		return null;
 	}
 
-	const { instanceToken } = options.beforeNetwork?.instanceToken
+	const { instanceToken } = options.beforeNetwork?.instanceToken && !(options.parameters?.native || options.parameters?.childSession || options.parameters?.parentSession === false)
 		? options.beforeNetwork
 		: await getSession(site, options.parameters, url);
 

src/scrapers/gamma.js

@@ -21,6 +21,10 @@ async function scrapeApiReleases(json, site, options) {
 			return acc;
 		}
 
+		if (options.parameters?.filterChannel && scene.mainChannel?.id !== options.parameters.filterChannel) {
+			return acc;
+		}
+
 		const release = {
 			entryId: scene.clip_id,
 			description: scene.description,
@@ -237,7 +241,7 @@ async function scrapeScene(data, site, options, movieScenes) {
 	release.entryId = data.clip_id || data.movie_id;
 	release.title = curateTitle(data.title, site);
 	release.duration = data.length;
-	release.date = (data.date && new Date(data.date * 1000)) || unprint.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
+	release.date = (data.date && new Date(data.date * 1000)) || unprint.extractDate(data.release_date || data.last_modified, 'YYYY-MM-DD');
 	release.director = data.directors[0]?.name || null;
 
 	release.actors = data.actors.map((actor) => ({

src/scrapers/vixen.js

@@ -502,6 +502,7 @@ async function fetchScene(url, channel, baseRelease, options) {
 	}
 
 	const res = await unprint.get(url, {
+		interface: 'request',
 		useBrowser: !!options.parameters?.useBrowser,
 	});
 

src/tools/gamma_banners.js

@@ -0,0 +1,136 @@
+'use strict';
+
+const unprint = require('unprint');
+const fs = require('fs');
+const { Readable } = require('stream');
+const { pipeline } = require('stream/promises');
+
+const knex = require('../knex');
+const argv = require('../argv');
+const slugify = require('../utils/slugify');
+
+const apiUrl = 'https://vjoc5ygk89-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20JavaScript%20(3.33.0)%3B%20Browser%20(lite)%3B%20react%20(16.8.6)%3B%20react-instantsearch%20(5.7.0)%3B%20JS%20Helper%20(2.28.1)&x-algolia-application-id=VJOC5YGK89&x-algolia-api-key=c5546bdfb4d3f31daf49ed3bb1463561';
+
+async function fetchBanners() {
+	const res = await unprint.post(
+		apiUrl,
+		{
+			requests: [
+				{
+					indexName: 'creatives',
+					params: new URLSearchParams({
+						hitsPerPage: 1000,
+						maxValuesPerFacet: 100,
+						page: 0,
+						filters: '(ProgramType:Legacy OR ProgramType:Internal) AND NOT OverlayActive:false',
+						facets: '["SceneActors","SceneCategories","ProgramName","Size","Niche","MediaExt","SiteTag","OverlayName"]',
+						facetFilters: `[["SiteTag:${argv.site}"],["MediaExt:jpg", "MediaExt:png", "MediaExt:gif"]]`,
+					}).toString(),
+				},
+			],
+		},
+		{
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				referer: 'https://creatives.gammae.com/',
+			},
+		},
+	);
+
+	if (res.ok && res.data.results[0]) {
+		return res.data.results[0].hits;
+	}
+
+	console.error(`Failed API request (${res.status}): ${res.body}`);
+
+	return null;
+}
+
+async function matchTags(rawTags) {
+	if (!rawTags) {
+		return [];
+	}
+
+	const tags = rawTags
+		.map((tag) => tag?.trim().match(/[a-z0-9()]+/ig)?.join(' ').toLowerCase())
+		.filter(Boolean);
+
+	const tagEntries = await knex('tags')
+		.select('tags.slug', 'aliases.slug as alias_slug')
+		.whereIn(knex.raw('lower(tags.name)'), tags)
+		.leftJoin('tags as aliases', 'aliases.id', 'tags.alias_for')
+		.orderByRaw('CASE WHEN tags.alias_for IS NOT NULL THEN aliases.priority ELSE tags.priority END DESC');
+
+	return tagEntries.map((tagEntry) => tagEntry.alias_slug || tagEntry.slug);
+}
+
+async function init() {
+	const banners = await fetchBanners();
+
+	if (!banners) {
+		return;
+	}
+
+	await banners.reduce(async (chain, banner) => {
+		await chain;
+
+		const channel = slugify(banner.SiteTag, '');
+		const url = unprint.prefixUrl(banner.MediaLocation || banner.CreativeURL, 'https://cdn.banhq.com');
+
+		if (!url) {
+			console.log('No URL found');
+			console.log(banner);
+			return;
+		}
+
+		const tags = await matchTags([
+			...banner.Tags?.map((tag) => tag.Value) || [],
+			...banner.SceneCategories || [],
+			banner.Niche,
+		].filter(Boolean));
+
+		const fileTags = tags.slice(0, 4).join('_');
+		const fileActors = banner.SceneActors?.slice(0, 2).map((actor) => slugify(actor, '_')).join('_');
+
+		// tags are unreliable and describe entire scene, not banner, don't include by default
+		const segments = [channel, banner.Width, banner.Height, banner.MediaID, argv.actors?.[0] !== false && fileActors].filter(Boolean);
+		const filename = `${segments.join('_')}${argv.tags && argv.tags ? `-${fileTags}` : ''}.${banner.MediaExt || 'jpg'}`;
+
+		const filepath = `/tmp/gamma/${channel}/${filename}`;
+
+		if (argv.inspect) {
+			console.log(banner);
+		}
+
+		if (argv.preview) {
+			console.log(`Preview ${url}: ${filepath}`);
+			return;
+		}
+
+		await fs.promises.mkdir(`/tmp/gamma/${channel}`, { recursive: true });
+
+		try {
+			const res = await fetch(url);
+
+			if (res.ok && res.body) {
+				const writer = fs.createWriteStream(filepath);
+
+				await pipeline(Readable.fromWeb(res.body), writer);
+
+				if (argv.actors) {
+					console.log(`Saved ${url} to ${filepath}`);
+				} else {
+					console.log(`Saved ${url} to ${filepath}, actors ${banner.SceneActors?.join(', ') || ''}`);
+				}
+			} else {
+				console.log(`Failed to fetch ${url} (${res.status})`);
+			}
+		} catch (error) {
+			console.log(`Failed to fetch ${url}: ${error.message}`);
+		}
+	}, Promise.resolve());
+
+	await knex.destroy();
+}
+
+init();

src/tools/huge-query.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const knex = require('../knex');
+
+async function init() {
+	const data = Array.from({ length: 100_000 }, (value, index) => ({
+		id: `test_affiliate_${index}`,
+	}));
+
+	await knex('affiliates').insert(data);
+
+	console.log('Done!');
+}
+
+init();

src/tools/manticore-scenes.js

@@ -41,7 +41,7 @@ async function fetchScenes() {
 			studios.name as studio_name,
 			grandparents.id as parent_network_id,
 			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
+			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
 			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
 			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
 			COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
@@ -136,6 +136,14 @@ async function init() {
 			 dupe_index int
 		)`);
 
+		await utilsApi.sql('drop table if exists scenes_tags');
+		await utilsApi.sql(`create table scenes_tags (
+			 id int,
+			 scene_id int,
+			 tag_id int,
+			 actor_id int
+		)`);
+
 		console.log('Recreated scenes table');
 		console.log('Fetching scenes from primary database');
 
@@ -143,49 +151,62 @@ async function init() {
 
 		console.log('Fetched scenes from primary database');
 
-		const docs = scenes.map((scene) => {
+		const docs = scenes.flatMap((scene) => {
 			const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
 			const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
 			const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
 
-			return {
-				replace: {
-					index: 'scenes',
-					id: scene.id,
-					doc: {
-						title: scene.title || undefined,
-						title_filtered: filteredTitle || undefined,
-						date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-						created_at: Math.round(scene.created_at.getTime() / 1000),
-						effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-						is_showcased: scene.showcased,
-						entry_id: scene.entry_id || undefined,
-						shoot_id: scene.shoot_id || undefined,
-						channel_id: scene.channel_id,
-						channel_slug: scene.channel_slug,
-						channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
-						network_id: scene.network_id || undefined,
-						network_slug: scene.network_slug || undefined,
-						network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
-						studio_id: scene.studio_id || undefined,
-						studio_slug: scene.studio_slug || undefined,
-						studio_name: scene.studio_name || undefined,
-						entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-						actor_ids: scene.actors.map((actor) => actor.f1),
-						actors: scene.actors.map((actor) => actor.f2).join(),
-						tag_ids: scene.tags.map((tag) => tag.f1),
-						tags: flatTags.join(' '),
-						movie_ids: scene.movies.map((movie) => movie.f1),
-						movies: scene.movies.map((movie) => movie.f2).join(' '),
-						serie_ids: scene.series.map((serie) => serie.f1),
-						series: scene.series.map((serie) => serie.f2).join(' '),
-						fingerprints: scene.fingerprints.join(' '),
-						meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
-						stashed: scene.stashed || 0,
-						dupe_index: scene.dupe_index || 0,
+			return [
+				{
+					replace: {
+						index: 'scenes',
+						id: scene.id,
+						doc: {
+							title: scene.title || undefined,
+							title_filtered: filteredTitle || undefined,
+							date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
+							created_at: Math.round(scene.created_at.getTime() / 1000),
+							effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
+							is_showcased: scene.showcased,
+							entry_id: scene.entry_id || undefined,
+							shoot_id: scene.shoot_id || undefined,
+							channel_id: scene.channel_id,
+							channel_slug: scene.channel_slug,
+							channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
+							network_id: scene.network_id || undefined,
+							network_slug: scene.network_slug || undefined,
+							network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
+							studio_id: scene.studio_id || undefined,
+							studio_slug: scene.studio_slug || undefined,
+							studio_name: scene.studio_name || undefined,
+							entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
+							actor_ids: scene.actors.map((actor) => actor.f1),
+							actors: scene.actors.map((actor) => actor.f2).join(),
+							tag_ids: scene.tags.map((tag) => tag.f1),
+							tags: flatTags.join(' '),
+							movie_ids: scene.movies.map((movie) => movie.f1),
+							movies: scene.movies.map((movie) => movie.f2).join(' '),
+							serie_ids: scene.series.map((serie) => serie.f1),
+							series: scene.series.map((serie) => serie.f2).join(' '),
+							fingerprints: scene.fingerprints.join(' '),
+							meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
+							stashed: scene.stashed || 0,
+							dupe_index: scene.dupe_index || 0,
+						},
 					},
 				},
-			};
+				...scene.tags.map((tag) => ({
+					replace: {
+						index: 'scenes_tags',
+						// id: scene.id,
+						doc: {
+							scene_id: scene.id,
+							tag_id: tag.f1,
+							actor_id: tag.f5,
+						},
+					},
+				})),
+			];
 		});
 
 		// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {

src/tools/manticore-stashes.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const config = require('config');
+const manticore = require('manticoresearch');
+
+const knex = require('../knex');
+const chunk = require('../utils/chunk');
+
+const mantiClient = new manticore.ApiClient();
+
+mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
+
+const utilsApi = new manticore.UtilsApi(mantiClient);
+const indexApi = new manticore.IndexApi(mantiClient);
+
+async function syncStashes(domain = 'scene') {
+	await utilsApi.sql(`truncate table ${domain}s_stashed`);
+
+	const stashes = await knex(`stashes_${domain}s`)
+		.select(
+			`stashes_${domain}s.id as stashed_id`,
+			`stashes_${domain}s.${domain}_id`,
+			'stashes.id as stash_id',
+			'stashes.user_id as user_id',
+			`stashes_${domain}s.created_at as created_at`,
+		)
+		.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
+
+	await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
+		await chain;
+
+		const stashDocs = stashChunk.map((stash) => ({
+			replace: {
+				index: `${domain}s_stashed`,
+				id: stash.stashed_id,
+				doc: {
+					[`${domain}_id`]: stash[`${domain}_id`],
+					stash_id: stash.stash_id,
+					user_id: stash.user_id,
+					created_at: Math.round(stash.created_at.getTime() / 1000),
+				},
+			},
+		}));
+
+		await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
+
+		console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
+	}, Promise.resolve());
+}
+
+async function init() {
+	await utilsApi.sql('drop table if exists scenes_stashed');
+
+	await utilsApi.sql(`create table if not exists scenes_stashed (
+		scene_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await utilsApi.sql('drop table if exists movies_stashed');
+
+	await utilsApi.sql(`create table if not exists movies_stashed (
+		movie_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await utilsApi.sql('drop table if exists actors_stashed');
+
+	await utilsApi.sql(`create table if not exists actors_stashed (
+		actor_id int,
+		stash_id int,
+		user_id int,
+		created_at timestamp
+	)`);
+
+	await syncStashes('scene');
+	await syncStashes('actor');
+	await syncStashes('movie');
+
+	console.log('Done!');
+
+	knex.destroy();
+}
+
+init();

src/update-search.js

@@ -16,14 +16,14 @@ async function updateManticoreStashedScenes(docs) {
 	await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
 		await chain;
 
-		const sceneIds = docsChunk.map((doc) => doc.replace.id);
+		const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
 
 		const stashes = await knex('stashes_scenes')
 			.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
 			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
 			.whereIn('scene_id', sceneIds);
 
-		const stashDocs = docsChunk.flatMap((doc) => {
+		const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
 			const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
 
 			if (sceneStashes.length === 0) {
@@ -50,6 +50,25 @@ async function updateManticoreStashedScenes(docs) {
 		if (stashDocs.length > 0) {
 			await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
 		}
+
+		const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
+
+		if (deleteSceneIds.length > 0) {
+			await indexApi.callDelete({
+				index: 'scenes_stashed',
+				query: {
+					bool: {
+						must: [
+							{
+								in: {
+									scene_id: deleteSceneIds,
+								},
+							},
+						],
+					},
+				},
+			});
+		}
 	}, Promise.resolve());
 }
 
@@ -128,9 +147,20 @@ async function updateManticoreSceneSearch(releaseIds) {
 			studios.showcased
     `, releaseIds && [releaseIds]);
 
-	// console.log(scenes.rows);
+	const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
+
+	const docs = releaseIds.map((sceneId) => {
+		const scene = scenesById[sceneId];
+
+		if (!scene) {
+			return {
+				delete: {
+					index: 'scenes',
+					id: sceneId,
+				},
+			};
+		}
 
-	const docs = scenes.rows.map((scene) => {
 		const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
 		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
 		const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
@@ -291,7 +321,20 @@ async function updateManticoreMovieSearch(movieIds) {
 			movies_covers.*
     `, movieIds && [movieIds]);
 
-	const docs = movies.rows.map((movie) => {
+	const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
+
+	const docs = movieIds.map((movieId) => {
+		const movie = moviesById[movieId];
+
+		if (!movie) {
+			return {
+				delete: {
+					index: 'movies',
+					id: movieId,
+				},
+			};
+		}
+
 		const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
 			id: tag.f1,
 			name: tag.f2,