Added abilities seed file to repo.

config/default.js

@@ -199,6 +199,12 @@ module.exports = {
 			// source: 'http://nsfw.unknown.name/random',
 		},
 	},
+	webApi: {
+		enabled: true,
+		address: 'http://localhost:5100/api',
+		apiUserId: 1,
+		apiKey: null,
+	},
 	proxy: {
 		enable: false,
 		test: 'https://api.ipify.org?format=json',

migrations/20260520044355_actors_unique.js

@@ -0,0 +1,59 @@
+exports.up = async function(knex) {
+	await knex.raw(`
+		DROP INDEX unique_actor_slugs;
+		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
+	`);
+
+	await knex.schema.alterTable('actors', (table) => {
+		table.boolean('allow_global_match');
+		table.text('comment');
+	});
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ plainUrls: true },
+		]))
+		.where('role', 'admin');
+};
+
+exports.down = async function(knex) {
+	const dupes = await knex('actors')
+		.select('name', 'slug', knex.raw('count(*) as count'))
+		.whereNull('entity_id')
+		.groupBy('name', 'slug')
+		.havingRaw('count(*) > 1')
+		.orderBy('count', 'desc');
+
+	if (dupes.length > 0) {
+		console.log('DUPES\n', dupes.map((actor) => `${actor.name} ${actor.slug} ${actor.count}`).join('\n'));
+	}
+
+	await knex.raw(`
+		DROP INDEX unique_actor_slugs;
+		CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
+	`);
+
+	await knex.schema.alterTable('actors', (table) => {
+		table.dropColumn('allow_global_match');
+		table.dropColumn('comment');
+	});
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ plainUrls: true },
+		]))
+		.where('role', 'admin');
+};

migrations/20260608053154_sync_abilities.js

@@ -0,0 +1,87 @@
+exports.up = async function(knex) {
+	await knex.schema.createTable('sync', (table) => {
+		table.increments('id');
+
+		table.string('domain');
+		table.specificType('item_ids', 'integer array');
+
+		table.text('comment');
+
+		table.datetime('created_at')
+			.defaultTo(knex.fn.now());
+	});
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ subject: 'sync' },
+			{ subject: 'plainUrls' },
+		]))
+		.where('role', 'admin');
+
+	await knex.raw(`
+		DROP TABLE IF EXISTS releases_search CASCADE;
+		DROP TABLE IF EXISTS movies_search CASCADE;
+		DROP TABLE IF EXISTS series_search CASCADE;
+
+		DROP TABLE IF EXISTS releases_search_results CASCADE;
+		DROP TABLE IF EXISTS movies_search_results CASCADE;
+	`);
+};
+
+exports.down = async function(knex) {
+	await knex.schema.dropTable('sync');
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ plainUrls: true },
+		]))
+		.where('role', 'admin');
+
+	await knex.schema.createTable('releases_search', (table) => {
+		table.integer('release_id', 16)
+			.references('id')
+			.inTable('releases')
+			.onDelete('cascade');
+	});
+
+	await knex.schema.createTable('movies_search', (table) => {
+		table.integer('movie_id', 16)
+			.references('id')
+			.inTable('movies')
+			.onDelete('cascade');
+	});
+
+	await knex.schema.createTable('series_search', (table) => {
+		table.integer('serie_id', 16)
+			.references('id')
+			.inTable('series')
+			.onDelete('cascade');
+	});
+
+	await knex.raw(`
+		ALTER TABLE releases_search ADD COLUMN document tsvector;
+		ALTER TABLE movies_search ADD COLUMN document tsvector;
+		ALTER TABLE series_search ADD COLUMN document tsvector;
+
+		CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
+		CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
+		CREATE INDEX releases_search_index ON releases_search USING GIN (document);
+		CREATE INDEX movies_search_index ON movies_search USING GIN (document);
+		CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
+		CREATE INDEX series_search_index ON series_search USING GIN (document);
+	`);
+};

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.252.9",
+    "version": "1.253.6",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.252.9",
+            "version": "1.253.6",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",
@@ -95,7 +95,7 @@
                 "tunnel": "0.0.6",
                 "ua-parser-js": "^1.0.37",
                 "undici": "^7.24.7",
-                "unprint": "^0.19.13",
+                "unprint": "^0.19.20",
                 "url-pattern": "^1.0.3",
                 "v-tooltip": "^2.1.3",
                 "video.js": "^8.6.1",
@@ -20844,9 +20844,9 @@
             }
         },
         "node_modules/unprint": {
-            "version": "0.19.13",
-            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.13.tgz",
-            "integrity": "sha512-HPNCQn2CziiGeK0JSZg/5E+G2prHme+8lDojxd16wUwSQ0mgW4nZq4LOuVMIRRAFm1M1nkju0oMIdsj4uRFASw==",
+            "version": "0.19.20",
+            "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.20.tgz",
+            "integrity": "sha512-Z1SVqsbs+CRS5VIRKKVmCYoki8q8gghY65lY+epxK4vMdT9rlouC9YGpHk0J9FusH0GQPIdG9MiWHIvAMGE3SA==",
             "dependencies": {
                 "bottleneck": "^2.19.5",
                 "cookie": "^1.1.1",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.252.9",
+    "version": "1.253.6",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {
@@ -154,7 +154,7 @@
         "tunnel": "0.0.6",
         "ua-parser-js": "^1.0.37",
         "undici": "^7.24.7",
-        "unprint": "^0.19.13",
+        "unprint": "^0.19.20",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.1.3",
         "video.js": "^8.6.1",

seeds/00_tags.js

@@ -647,6 +647,7 @@ const tags = [
 	{
 		name: 'MFF threesome',
 		slug: 'mff',
+		implies: ['threesome'],
 		description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
 		group: 'group',
 	},
@@ -705,6 +706,10 @@ const tags = [
 		name: 'gloryhole',
 		slug: 'gloryhole',
 	},
+	{
+		name: 'female gloryhole',
+		slug: 'female-gloryhole',
+	},
 	{
 		name: 'gonzo',
 		slug: 'gonzo',
@@ -826,6 +831,7 @@ const tags = [
 	{
 		name: 'MFM threesome',
 		slug: 'mfm',
+		implies: ['threesome'],
 		description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
 		group: 'group',
 	},
@@ -1414,6 +1420,10 @@ const tags = [
 		name: 'interview',
 		slug: 'interview',
 	},
+	{
+		name: 'pregnant',
+		slug: 'pregnant',
+	},
 ];
 
 const aliases = [
@@ -3032,6 +3042,58 @@ const aliases = [
 		name: 't---y f--k',
 		for: 'titty-fucking',
 	},
+	{
+		name: 'thresome',
+		for: 'threesome',
+	},
+	{
+		name: 'fuck',
+		for: 'sex',
+	},
+	{
+		name: 'suck',
+		for: 'blowjob',
+	},
+	{
+		name: 'analfist',
+		for: 'anal-fisting',
+	},
+	{
+		name: 'fivesome',
+		for: 'orgy',
+	},
+	{
+		name: 'fucking machine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuck machine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuckmashine',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'fuck saw',
+		for: 'machine-dildo',
+	},
+	{
+		name: 'dirtytalk',
+		for: 'dirty-talk',
+	},
+	{
+		name: 'stepmom',
+		for: 'family',
+	},
+	{
+		name: 'ass2mouth',
+		for: 'atm',
+	},
+	{
+		name: 'fist',
+		for: 'fisting',
+	},
 ];
 
 const priorities = [ // higher index is higher priority

seeds/01_networks.js

@@ -239,6 +239,11 @@ const networks = [
 			// scene: false,
 		},
 	},
+	{
+		slug: 'acam',
+		name: 'A-Cam',
+		hasLogo: false,
+	},
 	{
 		slug: 'amateurallure',
 		name: 'Amateur Allure',

seeds/02_sites.js

@@ -9451,9 +9451,10 @@ const sites = [
 		parent: 'nubiles',
 	},
 	{
-		slug: 'lilsis',
-		name: 'Lil Sis',
-		url: 'https://lilsis.com',
+		slug: 'realitysis',
+		name: 'Reality Sis',
+		url: 'https://realitysis.com',
+		alias: ['lil sis'],
 		tags: ['family'],
 		parent: 'nubiles',
 	},
@@ -9464,13 +9465,6 @@ const sites = [
 		tags: ['cheating', 'family'],
 		parent: 'nubiles',
 	},
-	{
-		slug: 'realitysis',
-		name: 'Reality Sis',
-		url: 'https://realitysis.com',
-		tags: ['family'],
-		parent: 'nubiles',
-	},
 	{
 		slug: 'shesbreedingmaterial',
 		name: 'She\'s Breeding Material',
@@ -14683,6 +14677,47 @@ const sites = [
 		description: 'BustyOnes.com bringing you the most beautiful big breasts in the world! The hottest women alive showcasing their fantastic tits.',
 		parent: 'twistys',
 	},
+	// A-CAM / VILDE
+	{
+		name: 'Vilde',
+		slug: 'vilde',
+		alias: ['vilde tv', 'johan vilde'],
+		url: 'https://vilde.tv',
+		parent: 'acam',
+		independent: true,
+		parameters: {
+			languageUrl: 'https://vilde.tv/select-language',
+			languageKey: 'select_lang',
+			language: 'English',
+		},
+	},
+	{
+		name: 'Anal Hooked',
+		slug: 'analhooked',
+		url: 'https://analhooked.com',
+		parent: 'acam',
+		tags: ['anal'],
+		independent: true,
+		parameters: {
+			layout: 'hooked',
+			staticUrl: true,
+			languageUrl: 'https://analhooked.com/change-option',
+			languageKey: 'select_language',
+			language: 'english',
+		},
+	},
+	{
+		name: 'Channel Anal',
+		slug: 'channelanal',
+		alias: ['kanal anal'],
+		url: 'https://channelanal.com',
+		parent: 'acam',
+		independent: true,
+		tags: ['anal'],
+		parameters: {
+			layout: 'kanal',
+		},
+	},
 	// VIP SEX VAULT
 	{
 		name: 'Los Consoladores',
@@ -15245,6 +15280,16 @@ sites.reduce((acc, site) => {
 
 /* eslint-disable max-len */
 exports.seed = async (knex) => {
+	sites.reduce((acc, channel) => {
+		if (acc.has(channel.slug)) {
+			console.log('DUPLICATE', channel.slug);
+		} else {
+			acc.add(channel.slug);
+		}
+
+		return acc;
+	}, new Set());
+
 	await Promise.all(sites.map(async (channel) => {
 		if (channel.rename) {
 			await knex('entities')

seeds/08_abilities.js

@@ -0,0 +1,28 @@
+exports.seed = async (knex) => {
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ subject: 'sync' },
+			{ subject: 'plainUrls' },
+		]))
+		.where('role', 'admin');
+
+	await knex('users_roles')
+		.update('abilities', JSON.stringify([
+			{ subject: 'scene', action: 'create' },
+			{ subject: 'scene', action: 'update' },
+			{ subject: 'scene', action: 'delete' },
+			{ subject: 'actor', action: 'create' },
+			{ subject: 'actor', action: 'update' },
+			{ subject: 'actor', action: 'delete' },
+			{ subject: 'actor', action: 'merge' },
+			{ subject: 'plainUrls' },
+		]))
+		.where('role', 'editor');
+};

src/actors.js

@@ -30,6 +30,7 @@ const { toBaseReleases } = require('./deep');
 const { associateAvatars, flushOrphanedMedia } = require('./media');
 const { fetchEntitiesBySlug } = require('./entities');
 const { deleteScenes } = require('./releases');
+const { updateActorSearch } = require('./update-search');
 
 const actorsCommon = import('../common/actors.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
 const geoCommon = import('../common/geo.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
@@ -912,7 +913,7 @@ async function getOrCreateActors(baseActors, batchId) {
 		.whereRaw(`
 			actors.slug = base_actors.slug
 			AND actors.entity_id IS NULL
-			AND NOT base_actors.collision_likely
+			AND (NOT base_actors.collision_likely OR actors.allow_global_match)
 		`)
 		.orWhereRaw(`
 			actors.slug = base_actors.slug
@@ -980,7 +981,8 @@ async function getOrCreateActors(baseActors, batchId) {
 
 	await storeProfiles(newActorProfiles);
 
-	if (Array.isArray(newActors)) {
+	if (Array.isArray(newActors) && newActors.length > 0) {
+		await updateActorSearch(newActors.map((actor) => actor.id));
 		return newActors.concat(existingActors);
 	}
 

src/media.js

@@ -28,7 +28,6 @@ const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 const { fetchEntityReleaseIds } = require('./entity-releases');
 
-// const pipeline = util.promisify(stream.pipeline);
 const streamQueue = taskQueue();
 
 const s3 = new S3Client({
@@ -516,16 +515,6 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
 			writeLazy(image, lazypath, info),
 		]);
 
-		/*
-		if (isProcessed) {
-			// file already stored, remove temporary file
-			await fsPromises.unlink(media.file.path);
-		} else {
-			// image not processed, simply move temporary file to final location
-			await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
-		}
-		*/
-
 		await fsPromises.unlink(media.file.path);
 
 		if (config.s3.enabled) {
@@ -731,31 +720,31 @@ async function fetchSource(source, baseMedia) {
 	}
 
 	async function attempt(attempts = 1) {
-		const hasher = new blake2.Hash('blake2b', { digestLength: 24 });
-		let hasherReady = true;
-		hasher.setEncoding('hex');
+		const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
+		let tempFileTarget;
+
+		const hasher = blake2.createHash('blake2b', { digestLength: 24 });
+
+		let size = 0;
+
+		const hashStream = new stream.Transform({
+			transform(streamChunk, _encoding, callback) {
+				size += streamChunk.length;
+				hasher.update(streamChunk);
+				this.push(streamChunk);
+				callback();
+			},
+		});
 
 		try {
-			const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
-			const tempFileTarget = fs.createWriteStream(tempFilePath);
-			const hashStream = new stream.PassThrough();
-			let size = 0;
-
-			hashStream.on('data', (streamChunk) => {
-				size += streamChunk.length;
-
-				if (hasherReady) {
-					hasher.write(streamChunk);
-				}
-			});
+			tempFileTarget = fs.createWriteStream(tempFilePath);
 
 			const { mimetype } = source.stream
 				? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
 				: await fetchHttpSource(source, tempFileTarget, hashStream);
 
-			hasher.end();
+			const hash = hasher.digest('hex');
 
-			const hash = hasher.read();
 			const [type, subtype] = mimetype.split('/');
 			const extension = mime.getExtension(mimetype);
 
@@ -778,8 +767,10 @@ async function fetchSource(source, baseMedia) {
 				},
 			};
 		} catch (error) {
-			hasherReady = false;
-			hasher.end();
+			// hasherReady = false;
+			// hasher.end();
+			hashStream.destroy();
+			tempFileTarget?.destroy();
 
 			if (error.code !== 'VERIFY_TYPE') {
 				logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);

src/scrapers/acam.js

@@ -0,0 +1,250 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const slugify = require('../utils/slugify');
+
+function extractEntryId(posters) {
+	try {
+		const poster = [].concat(posters).filter(Boolean)[0];
+		return slugify(new URL(poster.src || poster).pathname.match(/\/images\/(.*?)\.[a-z]{3,4}/i)?.[1]?.replace(/smak.*/i, ''), '');
+	} catch (error) {
+		return null;
+	}
+}
+
+function extractTags(title, titleComment) {
+	if (!title && !titleComment) {
+		return [];
+	}
+
+	if (titleComment?.includes('<i>')) {
+		const tagsMatch = titleComment.match(/<i>(.*?)<\/i>/)?.[1];
+
+		if (tagsMatch) {
+			return tagsMatch.split('-').map((tag) => tag.trim().toLowerCase());
+		}
+	}
+
+	const firstTagIndex = title.match(/[A-Z]{2}/)?.index;
+
+	if (firstTagIndex) {
+		const tagSection = title
+			.slice(firstTagIndex)
+			.match(/([A-Z0-9\s]{2,})/g);
+
+		if (tagSection) {
+			return tagSection
+				.map((tag) => tag.trim().toLowerCase())
+				.filter(Boolean) || [];
+		}
+	}
+
+	return [];
+}
+
+// derived photo is usually uncensored and preferred as poster, but not guaranteed to exist, so fall back to original image
+function getPhotos(poster) {
+	const photo = poster?.replace(/(s[ma]{2}kprov\d*)|([._]preview)/i, ''); // sic
+
+	if (photo === poster) {
+		return {
+			poster: {
+				src: poster,
+				verifyType: 'image',
+			},
+			photos: [],
+		};
+	}
+
+	return {
+		poster: [{
+			src: photo,
+			verifyType: 'image',
+		}, {
+			src: poster,
+			verifyType: 'image',
+		}],
+		photos: [{
+			src: poster,
+			verifyType: 'image',
+		}],
+	};
+}
+
+function scrapeAll(scenes, channel, parameters) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		// Vilde URLs are temporary tokens for some reason, seem to be handled entirely back-end
+		const url = query.url('a[href*="/show-video"]');
+
+		release.token = new URL(url).pathname.match(/\/show-video\/([a-z0-9]+)/)?.[1];
+		release.forceDeep = true;
+
+		release.title = query.content('a h5, .product-content p, .video_text');
+		release.tags = extractTags(release.title, query.content('//a/comment()'));
+
+		const { poster, photos } = getPhotos(query.img('img[src*="/videos/images"], img[src*="/uploads/images"]'));
+
+		release.poster = poster;
+		release.photos = photos;
+
+		if (parameters.staticUrl) {
+			release.url = url;
+			release.entryId = release.token;
+		} else {
+			release.entryId = extractEntryId(release.poster);
+		}
+
+		return release;
+	});
+}
+
+async function setLanguage(parameters) {
+	if (parameters.languageUrl) {
+		const langRes = await unprint.post(parameters.languageUrl, {
+			[parameters.languageKey || 'select_language']: parameters.language || 'english',
+		}, {
+			form: true,
+		});
+
+		return langRes.cookies;
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.post(`${channel.origin}/pagination`, {
+		i: page,
+		status: true,
+	}, {
+		selectAll: '.movi-area',
+		form: true,
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestHooked(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.get(`${channel.origin}/Welcome/index/${(page - 1) * 9}`, {
+		selectAll: '.product-main',
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestKanal(channel, page = 1, { parameters }) {
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.post(`${channel.origin}/pagination`, {
+		k: page,
+		hidden_page_no: page - 1,
+		status: true,
+	}, {
+		selectAll: '.video_bx',
+		form: true,
+		cookies,
+	});
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel, parameters);
+	}
+
+	return res.status;
+}
+
+function scrapeScene({ query }, { url, baseRelease, parameters }) {
+	const release = {};
+
+	// URL is temporary token
+	if (!query.exists('.login-sec.for-browser, .video-description, .video_co_title')) {
+		// URL likely expired, still returns 200
+		return null;
+	}
+
+	if (query.exists('.video-description')) {
+		const descriptions = query.contents('.video-description p').filter(Boolean);
+
+		release.title = descriptions[0];
+		release.description = descriptions.slice(1).join(' ') || null;
+	} else {
+		release.title = query.content('.login-sec.for-browser h3, .video_co_title h3');
+		release.description = query.contents('.login-sec.for-browser h3 ~ *').join(' ') || null;
+	}
+
+	release.tags = extractTags(release.title);
+
+	const { poster, photos } = getPhotos(query.poster('.play_video_cont video'));
+
+	release.poster = poster;
+	release.photos = photos;
+
+	release.trailer = query.all('.play_video_cont source')
+		.map((videoEl) => ({
+			src: unprint.query.url(videoEl, null, { attribute: 'src' }),
+			quality: unprint.query.number(videoEl, null, { attribute: 'size' }),
+			referer: url,
+		}))
+		.toSorted((videoA, videoB) => videoB.quality - videoA.quality);
+
+	if (parameters.staticUrl) {
+		release.url = url;
+		release.entryId = baseRelease?.token || new URL(url).pathname.match(/\/show-video\/([a-z0-9]+)/)?.[1];
+	} else {
+		release.entryId = extractEntryId(release.poster);
+	}
+
+	return release;
+}
+
+async function fetchScene(baseUrl, entity, baseRelease, { parameters }) {
+	const url = baseUrl || (baseRelease?.token && `${entity.origin}/show-video/${baseRelease.token}`) || null;
+
+	if (!url) {
+		return null;
+	}
+
+	const cookies = await setLanguage(parameters);
+
+	const res = await unprint.get(url, {
+		headers: {
+			'accept-language': 'en-US,en',
+		},
+		cookies,
+	});
+
+	if (res.ok || res.status === 500) { // Anal Hooked returns 500 for valid scene pages
+		return scrapeScene(res.context, { url, baseRelease, parameters });
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	hooked: {
+		fetchLatest: fetchLatestHooked,
+		fetchScene,
+	},
+	kanal: {
+		fetchLatest: fetchLatestKanal,
+		fetchScene,
+	},
+};

src/scrapers/julesjordan.js

@@ -256,13 +256,15 @@ async function scrapeScene({ html, query }, context) {
 		})));
 	}
 
-	if (query.exists('.player-scene-description a[href*="/dvd"]')) {
+	const movieQuery = '//div[@class="meta-item"][div[@class="lbl" and text()="Movie"]]/div[@class="val"]/a';
+
+	if (query.exists(movieQuery)) {
 		release.movie = {
-			url: query.url('.player-scene-description a[href*="/dvd"]'),
-			title: query.content('.player-scene-description a[href*="/dvd"]'),
+			url: query.url(movieQuery),
+			title: query.content(movieQuery),
 		};
 
-		release.movie.entryId = new URL(release.movie.url).pathname.split('/').slice(-1)[0]?.replace('.html', '').toLowerCase();
+		release.movie.entryId = new URL(release.movie.url).pathname.match(/\/dvds\/(.*?).html/)?.[1];
 	}
 
 	release.stars = query.number('.avg_rating');
@@ -275,32 +277,35 @@ async function scrapeScene({ html, query }, context) {
 function scrapeMovie({ query }, { url }) {
 	const movie = {};
 
-	movie.entryId = new URL(url).pathname.split('/').slice(-1)[0]?.replace('.html', '').toLowerCase();
-	movie.title = query.attribute('meta[property="og:title"]', 'content');
+	movie.entryId = new URL(url).pathname.match(/\/dvds\/(.*?).html/)?.[1];
+	movie.title = query.attribute('meta[property="og:title"]', 'content')?.replace(/\s*-\s*jules jordan/i, '');
 
 	movie.covers = [query.img('img.dvd_box')]; // -2x etc is likely upscaled
 
-	const sceneTitles = query.contents('.title-heading-content-black-dvd');
+	const descriptionDate = query.content('meta[name="description"]', { attribute: 'content' })?.match(/released (\w+ \d{4})/i)?.[1];
 
-	const scenes = query.all('.grid-container-scene').map((sceneEl, index) => {
+	if (descriptionDate) {
+		movie.date = unprint.extractDate(descriptionDate, 'MMMM YYYY', { match: null });
+		movie.datePrecision = 'month';
+	}
+
+	const scenes = query.all('.dvd-scene').map((sceneEl) => {
 		const scene = {};
 
-		scene.url = unprint.query.url(sceneEl, 'a[href*="/scenes"]');
-		scene.title = sceneTitles[index];
-
-		scene.date = unprint.query.date(sceneEl, '//span[contains(@class, "dvd-scene-description") and span[contains(text(), "Date")]]', 'MM/DD/YYYY');
-		scene.actors = unprint.query.contents(sceneEl, '.update_models a');
-
+		scene.url = unprint.query.url(sceneEl, '.dvd-watch-btn');
 		scene.entryId = getEntryId(scene.url);
 
+		scene.title = unprint.query.content(sceneEl, '.dvd-hero-title');
+		scene.date = unprint.query.date(sceneEl, '.dvd-hero-date', 'MM/DD/YYYY');
+		scene.actors = unprint.query.contents(sceneEl, '.dvd-hero .update_models a');
+
+		scene.photos = unprint.query.imgs(sceneEl, '.dvd-img-strip img');
+
 		return scene;
 	});
 
 	movie.scenes = scenes?.sort((sceneA, sceneB) => sceneA.date - sceneB.date);
 
-	movie.date = movie.scenes?.[0]?.date;
-	movie.datePrecision = 'month';
-
 	return movie;
 }
 

src/scrapers/kink.js

@@ -18,7 +18,7 @@ function scrapeAll(scenes, entity) {
 		release.entryId = release.shootId;
 
 		release.title = query.content('.card-body a[href*="/shoot"]').trim();
-		release.date = query.date('small > span', 'MMM D, YYYY');
+		release.date = query.date('.small > span', 'MMM D, YYYY');
 
 		release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
 			name: unprint.query.content(actorEl),
@@ -54,7 +54,11 @@ function scrapeAll(scenes, entity) {
 			// no photos
 		}
 
-		release.trailer = `https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`;
+		release.trailer = [
+			query.video('.ratio-thumbnail img', { attribute: 'data-trailer-url' }),
+			`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
+			`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
+		];
 
 		release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
 		release.rating = query.number('.thumb-up') / 10;
@@ -64,10 +68,13 @@ function scrapeAll(scenes, entity) {
 }
 
 async function fetchLatest(channel, page = 1) {
-	const url = `${channel.parent.url}/search?type=shoots&channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
+	const url = `${channel.parent.url}/shoots?channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
 
 	const res = await unprint.browser(url, {
 		selectAll: '.container .card',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -129,6 +136,7 @@ function scrapeScene({ query }, url, entity) {
 			src: source.url,
 			quality: source.resolution,
 		})) || []),
+		`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
 		`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
 	];
 
@@ -143,7 +151,11 @@ function scrapeScene({ query }, url, entity) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await unprint.browser(url);
+	const res = await unprint.browser(url, {
+		cookies: {
+			age_gate_accepted: '1',
+		},
+	});
 
 	if (res.status === 200) {
 		const scene = scrapeScene(res.context, url, channel);
@@ -197,6 +209,9 @@ async function getActorUrl({ name: actorName, url }, networkUrl) {
 	// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
 	const res = await unprint.get(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`, {
 		interface: 'request',
+		cookies: {
+			age_gate_accepted: '1',
+		},
 	});
 
 	if (res.status === 200) {
@@ -217,7 +232,11 @@ async function fetchProfile(actor, entity) {
 	const actorUrl = await getActorUrl(actor, networkUrl);
 
 	if (actorUrl) {
-		const actorRes = await unprint.browser(actorUrl);
+		const actorRes = await unprint.browser(actorUrl, {
+			cookies: {
+				age_gate_accepted: '1',
+			},
+		});
 
 		if (actorRes.status === 200) {
 			return scrapeProfile(actorRes.context, actorUrl);

src/scrapers/nubiles.js

@@ -21,9 +21,9 @@ function stripQuery(link) {
 }
 
 async function getPhotos(albumUrl) {
-	const res = await unprint.get(albumUrl, {
+	// aggressive Turnstile protection calls for browser
+	const res = await unprint.browser(albumUrl, {
 		selectAll: '.photo-thumb',
-		interface: 'request',
 	});
 
 	return res.ok

src/scrapers/releases.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const acam = require('./acam');
 const adultempire = require('./adultempire');
 const angelogodshackoriginal = require('./angelogodshackoriginal');
 // const archangel = require('./archangel');
@@ -96,6 +97,7 @@ module.exports = {
 	freeuse: teamskeet,
 	familystrokes: teamskeet,
 	// etc
+	acam,
 	analvids: pornbox,
 	pornbox,
 	kellymadison,

src/store-releases.js

@@ -344,6 +344,8 @@ async function associateMovieScenes(movies, movieScenes) {
 		.filter(Boolean);
 
 	await bulkInsert('movies_scenes', associations, false);
+
+	await updateSceneSearch(movieScenes.map((scene) => scene.id));
 }
 
 async function associateSerieScenes(series, serieScenes) {

src/tools/manticore-actors.js

@@ -1,107 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-
-const knex = require('../knex');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const searchApi = new manticore.SearchApi(mantiClient);
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchActors() {
-	// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
-	const actors = await knex.raw(`
-        SELECT
-			actors.*,
-			actors_meta.*,
-			date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
-        FROM actors
-		LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id
-	`);
-
-	return actors.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists actors');
-		await utilsApi.sql(`create table actors(
-			 id int,
-			 name text,
-			 slug string,
-			 gender string,
-			 date_of_birth timestamp,
-			 country string,
-			 has_avatar bool,
-			 mass int,
-			 height int,
-			 cup string,
-			 natural_boobs int,
-			 penis_length int,
-			 penis_girth int,
-			 stashed int,
-			 scenes int
-		) min_prefix_len = '3'`);
-
-		const actors = await fetchActors();
-
-		const docs = actors.map((actor) => ({
-			insert: {
-				index: 'actors',
-				id: actor.id,
-				doc: {
-					name: actor.name,
-					slug: actor.slug,
-					gender: actor.gender || undefined,
-					date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
-					has_avatar: !!actor.avatar_media_id,
-					country: actor.birth_country_alpha2 || undefined,
-					height: actor.height || undefined,
-					mass: actor.weight || undefined, // weight is a reserved keyword in manticore
-					cup: actor.cup || undefined,
-					natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
-					penis_length: actor.penis_length || undefined,
-					penis_girth: actor.penis_girth || undefined,
-					stashed: actor.stashed || 0,
-					scenes: actor.scenes || 0,
-				},
-			},
-		}));
-
-		const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
-			console.log(error);
-		});
-
-		console.log('data', data);
-		knex.destroy();
-
-		return;
-	}
-
-	const result = await searchApi.search({
-		index: 'actors',
-		query: {
-			equals: {
-				has_avatar: 1,
-			},
-		},
-		limit: 3,
-		sort: [{ slug: 'asc' }],
-	});
-
-	console.log(result);
-	console.log(result.hits?.hits);
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-movies.js

@@ -1,165 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-const { format } = require('date-fns');
-
-const knex = require('../knex');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-// const searchApi = new manticore.SearchApi(mantiClient);
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchMovies() {
-	const movies = await knex.raw(`
-        SELECT
-            movies.id AS id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			movies_covers IS NOT NULL as has_cover,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
-			row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
-        FROM movies
-		LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
-		LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
-		LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
-        LEFT JOIN entities ON movies.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
-        LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
-        LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
-        GROUP BY
-            movies.id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			movies_meta.stashed_scenes,
-			movies_meta.stashed_total,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			movies_covers.*
-	`);
-
-	return movies.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists movies');
-		await utilsApi.sql(`create table movies (
-			 id int,
-			 title text,
-			 title_filtered text,
-			 channel_id int,
-			 channel_name text,
-			 channel_slug text,
-			 network_id int,
-			 network_name text,
-			 network_slug text,
-			 entity_ids multi,
-			 actor_ids multi,
-			 actors text,
-			 tag_ids multi,
-			 tags text,
-			 meta text,
-			 date timestamp,
-			 has_cover bool,
-			 created_at timestamp,
-			 effective_date timestamp,
-			 stashed int,
-			 stashed_scenes int,
-			 stashed_total int,
-			 dupe_index int
-		)`);
-
-		const movies = await fetchMovies();
-
-		console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index));
-
-		const docs = movies.map((movie) => {
-			const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
-				id: tag.f1,
-				name: tag.f2,
-				priority: tag.f3,
-				alias: tag.f4,
-			}])));
-
-			const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-			const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-			const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
-
-			return {
-				replace: {
-					index: 'movies',
-					id: movie.id,
-					doc: {
-						title: movie.title || undefined,
-						title_filtered: filteredTitle || undefined,
-						date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
-						created_at: Math.round(movie.created_at.getTime() / 1000),
-						effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
-						channel_id: movie.channel_id,
-						channel_slug: movie.channel_slug,
-						channel_name: movie.channel_name,
-						network_id: movie.network_id || undefined,
-						network_slug: movie.network_slug || undefined,
-						network_name: movie.network_name || undefined,
-						entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
-						actor_ids: movie.actors.map((actor) => actor.f1),
-						actors: movie.actors.map((actor) => actor.f2).join(),
-						tag_ids: combinedTags.map((tag) => tag.id),
-						tags: flatTags.join(' '),
-						has_cover: movie.has_cover,
-						meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined,
-						stashed: movie.stashed || 0,
-						stashed_scenes: movie.stashed_scenes || 0,
-						stashed_total: movie.stashed_total || 0,
-						dupe_index: movie.dupe_index || 0,
-					},
-				},
-			};
-		});
-
-		console.log(docs.map((doc) => doc.replace));
-
-		const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-
-		console.log('data', data);
-	}
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-scenes.js

@@ -1,228 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-const args = require('yargs').argv;
-const { format } = require('date-fns');
-
-const knex = require('../knex');
-const chunk = require('../utils/chunk');
-const filterTitle = require('../utils/filter-title');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-const update = args.update;
-
-async function fetchScenes() {
-	const scenes = await knex.raw(`
-        SELECT
-            releases.id AS id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.entry_id,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			entities.alias as channel_aliases,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			parents.alias as network_aliases,
-			studios.id as studio_id,
-			studios.slug as studio_slug,
-			studios.name as studio_name,
-			grandparents.id as parent_network_id,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
-			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
-			COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
-			studios.showcased IS NOT false
-			AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
-			AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
-			AND (releases_summaries.batch_showcased IS NOT false)
-			AS showcased,
-			row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
-        FROM releases
-		LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
-		LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
-        LEFT JOIN entities AS studios ON studios.id = releases.studio_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
-        LEFT JOIN movies ON movies.id = movies_scenes.movie_id
-        LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
-        LEFT JOIN series ON series.id = series_scenes.serie_id
-        GROUP BY
-            releases.id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.entry_id,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			releases_summaries.batch_showcased,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			grandparents.id,
-			studios.id,
-			studios.name,
-			studios.slug,
-			entities.showcased,
-			parents.showcased,
-			studios.showcased;
-	`);
-
-	return scenes.rows;
-}
-
-async function init() {
-	if (update) {
-		await utilsApi.sql('drop table if exists scenes');
-		await utilsApi.sql(`create table scenes (
-			 id int,
-			 title text,
-			 title_filtered text,
-			 entry_id text,
-			 shoot_id text,
-			 channel_id int,
-			 channel_name text,
-			 channel_slug text,
-			 network_id int,
-			 network_name text,
-			 network_slug text,
-			 studio_id int,
-			 studio_name text,
-			 studio_slug text,
-			 entity_ids multi,
-			 actor_ids multi,
-			 actors text,
-			 tag_ids multi,
-			 tags text,
-			 movie_ids multi,
-			 movies text,
-			 serie_ids multi,
-			 series text,
-			 meta text,
-			 date timestamp,
-			 fingerprints text,
-			 is_showcased bool,
-			 created_at timestamp,
-			 effective_date timestamp,
-			 stashed int,
-			 dupe_index int
-		)`);
-
-		await utilsApi.sql('drop table if exists scenes_tags');
-		await utilsApi.sql(`create table scenes_tags (
-			 id int,
-			 scene_id int,
-			 tag_id int,
-			 actor_id int
-		)`);
-
-		console.log('Recreated scenes table');
-		console.log('Fetching scenes from primary database');
-
-		const scenes = await fetchScenes();
-
-		console.log('Fetched scenes from primary database');
-
-		const docs = scenes.flatMap((scene) => {
-			const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-			const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-			const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
-
-			return [
-				{
-					replace: {
-						index: 'scenes',
-						id: scene.id,
-						doc: {
-							title: scene.title || undefined,
-							title_filtered: filteredTitle || undefined,
-							date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-							created_at: Math.round(scene.created_at.getTime() / 1000),
-							effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-							is_showcased: scene.showcased,
-							entry_id: scene.entry_id || undefined,
-							shoot_id: scene.shoot_id || undefined,
-							channel_id: scene.channel_id,
-							channel_slug: scene.channel_slug,
-							channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
-							network_id: scene.network_id || undefined,
-							network_slug: scene.network_slug || undefined,
-							network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
-							studio_id: scene.studio_id || undefined,
-							studio_slug: scene.studio_slug || undefined,
-							studio_name: scene.studio_name || undefined,
-							entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-							actor_ids: scene.actors.map((actor) => actor.f1),
-							actors: scene.actors.map((actor) => actor.f2).join(),
-							tag_ids: scene.tags.map((tag) => tag.f1),
-							tags: flatTags.join(' '),
-							movie_ids: scene.movies.map((movie) => movie.f1),
-							movies: scene.movies.map((movie) => movie.f2).join(' '),
-							serie_ids: scene.series.map((serie) => serie.f1),
-							series: scene.series.map((serie) => serie.f2).join(' '),
-							fingerprints: scene.fingerprints.join(' '),
-							meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
-							stashed: scene.stashed || 0,
-							dupe_index: scene.dupe_index || 0,
-						},
-					},
-				},
-				...scene.tags.map((tag) => ({
-					replace: {
-						index: 'scenes_tags',
-						// id: scene.id,
-						doc: {
-							scene_id: scene.id,
-							tag_id: tag.f1,
-							actor_id: tag.f5,
-						},
-					},
-				})),
-			];
-		});
-
-		// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
-		chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
-			const acc = await chain;
-			const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
-
-			console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
-
-			return acc.concat(data.items);
-		}, Promise.resolve([]));
-
-		// console.log('data', accData);
-	}
-
-	knex.destroy();
-}
-
-init();

src/tools/manticore-stashes.js

@@ -1,88 +0,0 @@
-'use strict';
-
-const config = require('config');
-const manticore = require('manticoresearch');
-
-const knex = require('../knex');
-const chunk = require('../utils/chunk');
-
-const mantiClient = new manticore.ApiClient();
-
-mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
-
-const utilsApi = new manticore.UtilsApi(mantiClient);
-const indexApi = new manticore.IndexApi(mantiClient);
-
-async function syncStashes(domain = 'scene') {
-	await utilsApi.sql(`truncate table ${domain}s_stashed`);
-
-	const stashes = await knex(`stashes_${domain}s`)
-		.select(
-			`stashes_${domain}s.id as stashed_id`,
-			`stashes_${domain}s.${domain}_id`,
-			'stashes.id as stash_id',
-			'stashes.user_id as user_id',
-			`stashes_${domain}s.created_at as created_at`,
-		)
-		.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
-
-	await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
-		await chain;
-
-		const stashDocs = stashChunk.map((stash) => ({
-			replace: {
-				index: `${domain}s_stashed`,
-				id: stash.stashed_id,
-				doc: {
-					[`${domain}_id`]: stash[`${domain}_id`],
-					stash_id: stash.stash_id,
-					user_id: stash.user_id,
-					created_at: Math.round(stash.created_at.getTime() / 1000),
-				},
-			},
-		}));
-
-		await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
-
-		console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
-	}, Promise.resolve());
-}
-
-async function init() {
-	await utilsApi.sql('drop table if exists scenes_stashed');
-
-	await utilsApi.sql(`create table if not exists scenes_stashed (
-		scene_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await utilsApi.sql('drop table if exists movies_stashed');
-
-	await utilsApi.sql(`create table if not exists movies_stashed (
-		movie_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await utilsApi.sql('drop table if exists actors_stashed');
-
-	await utilsApi.sql(`create table if not exists actors_stashed (
-		actor_id int,
-		stash_id int,
-		user_id int,
-		created_at timestamp
-	)`);
-
-	await syncStashes('scene');
-	await syncStashes('actor');
-	await syncStashes('movie');
-
-	console.log('Done!');
-
-	knex.destroy();
-}
-
-init();

src/tools/update-search.js

@@ -1,12 +0,0 @@
-'use strict';
-
-const { updateSceneSearch, updateMovieSearch } = require('../update-search');
-
-async function init() {
-	await updateSceneSearch();
-	await updateMovieSearch();
-
-	process.exit();
-}
-
-init();

src/update-search.js

@@ -1,436 +1,48 @@
 'use strict';
 
-const manticore = require('manticoresearch');
-const { format } = require('date-fns');
+const config = require('config');
+const unprint = require('unprint');
 
 const knex = require('./knex');
-const logger = require('./logger')(__filename);
-const bulkInsert = require('./utils/bulk-insert');
-const chunk = require('./utils/chunk');
-const filterTitle = require('./utils/filter-title');
 
-const mantiClient = new manticore.ApiClient();
-const indexApi = new manticore.IndexApi(mantiClient);
-
-async function updateManticoreStashedScenes(docs) {
-	await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
-		await chain;
-
-		const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
-
-		const stashes = await knex('stashes_scenes')
-			.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
-			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
-			.whereIn('scene_id', sceneIds);
-
-		const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
-			const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
-
-			if (sceneStashes.length === 0) {
-				return [];
-			}
-
-			const stashDoc = sceneStashes.map((stash) => ({
-				replace: {
-					index: 'scenes_stashed',
-					id: stash.stashed_id,
-					doc: {
-						// ...doc.replace.doc,
-						scene_id: doc.replace.id,
-						user_id: stash.user_id,
-						stash_id: stash.stash_id,
-						created_at: Math.round(stash.created_at.getTime() / 1000),
-					},
-				},
-			}));
-
-			return stashDoc;
-		});
-
-		if (stashDocs.length > 0) {
-			await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
-		}
-
-		const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
-
-		if (deleteSceneIds.length > 0) {
-			await indexApi.callDelete({
-				index: 'scenes_stashed',
-				query: {
-					bool: {
-						must: [
-							{
-								in: {
-									scene_id: deleteSceneIds,
-								},
-							},
-						],
-					},
-				},
-			});
-		}
-	}, Promise.resolve());
-}
-
-async function updateManticoreSceneSearch(releaseIds) {
-	logger.info(`Updating Manticore search documents for ${releaseIds ? releaseIds.length : 'all' } scenes`);
-
-	const scenes = await knex.raw(`
-        SELECT
-            releases.id AS id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			studios.id as studio_id,
-			studios.slug as studio_slug,
-			studios.name as studio_name,
-			grandparents.id as parent_network_id,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
-			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
-			studios.showcased IS NOT false
-			AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
-			AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
-			AND (releases_summaries.batch_showcased IS NOT false)
-			AS showcased,
-			row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
-        FROM releases
-		LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
-		LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
-        LEFT JOIN entities AS studios ON studios.id = releases.studio_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
-        LEFT JOIN movies ON movies.id = movies_scenes.movie_id
-        LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
-        LEFT JOIN series ON series.id = series_scenes.serie_id
-        ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
-        GROUP BY
-            releases.id,
-			releases.title,
-			releases.created_at,
-			releases.date,
-			releases.shoot_id,
-			scenes_meta.stashed,
-			releases_summaries.batch_showcased,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			entities.showcased,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			grandparents.id,
-			studios.id,
-			studios.name,
-			studios.slug,
-			parents.showcased,
-			studios.showcased
-    `, releaseIds && [releaseIds]);
-
-	const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
-
-	const docs = releaseIds.map((sceneId) => {
-		const scene = scenesById[sceneId];
-
-		if (!scene) {
-			return {
-				delete: {
-					index: 'scenes',
-					id: sceneId,
-				},
-			};
-		}
-
-		const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
-		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
-		const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
-
-		return {
-			replace: {
-				index: 'scenes',
-				id: scene.id,
-				doc: {
-					title: scene.title || undefined,
-					title_filtered: filteredTitle || undefined,
-					date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
-					created_at: Math.round(scene.created_at.getTime() / 1000),
-					effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
-					is_showcased: scene.showcased,
-					shoot_id: scene.shoot_id || undefined,
-					channel_id: scene.channel_id,
-					channel_slug: scene.channel_slug,
-					channel_name: scene.channel_name,
-					network_id: scene.network_id || undefined,
-					network_slug: scene.network_slug || undefined,
-					network_name: scene.network_name || undefined,
-					studio_id: scene.studio_id || undefined,
-					studio_slug: scene.studio_slug || undefined,
-					studio_name: scene.studio_name || undefined,
-					entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
-					actor_ids: scene.actors.map((actor) => actor.f1),
-					actors: scene.actors.map((actor) => actor.f2).join(),
-					tag_ids: scene.tags.map((tag) => tag.f1),
-					tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
-					movie_ids: scene.movies.map((movie) => movie.f1),
-					movies: scene.movies.map((movie) => movie.f2).join(' '),
-					serie_ids: scene.series.map((serie) => serie.f1),
-					series: scene.series.map((serie) => serie.f2).join(' '),
-					meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
-					stashed: scene.stashed || 0,
-					dupe_index: scene.dupe_index || 0,
-				},
-			},
-		};
-	});
-
-	if (docs.length === 0) {
+async function syncWeb(domain, ids) {
+	if (!ids || ids.length === 0) {
 		return;
 	}
 
-	await Promise.all([
-		indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')),
-		updateManticoreStashedScenes(docs),
-	]);
-}
+	await knex('sync').insert({ domain, item_ids: ids });
 
-async function updateSqlSceneSearch(releaseIds) {
-	logger.info(`Updating SQL search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
-
-	const documents = await knex.raw(`
-        SELECT
-            releases.id AS release_id,
-            TO_TSVECTOR(
-                'english',
-                COALESCE(releases.title, '') || ' ' ||
-                releases.entry_id || ' ' ||
-                entities.name || ' ' ||
-                entities.slug || ' ' ||
-                COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
-                COALESCE(parents.name, '') || ' ' ||
-                COALESCE(parents.slug, '') || ' ' ||
-                COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
-                COALESCE(releases.shoot_id, '') || ' ' ||
-                COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
-                STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
-                STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
-            ) as document
-        FROM releases
-        LEFT JOIN entities ON releases.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
-        GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
-    `, releaseIds && [releaseIds]);
-
-	if (documents.rows?.length > 0) {
-		await bulkInsert('releases_search', documents.rows, ['release_id']);
+	if (config.webApi.enabled) {
+		await unprint.post(`${config.webApi.address}/sync`, null, {
+			headers: {
+				'api-user': config.webApi.apiUserId,
+				'api-key': config.webApi.apiKey,
+			},
+		});
 	}
-
-	await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
 }
 
 async function updateSceneSearch(releaseIds) {
 	await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
+	await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
 
-	await updateSqlSceneSearch(releaseIds);
-	await updateManticoreSceneSearch(releaseIds);
-}
-
-async function updateManticoreMovieSearch(movieIds) {
-	logger.info(`Updating Manticore search documents for ${movieIds ? movieIds.length : 'all' } movies`);
-
-	const movies = await knex.raw(`
-        SELECT
-            movies.id AS id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			entities.id as channel_id,
-			entities.slug as channel_slug,
-			entities.name as channel_name,
-			parents.id as network_id,
-			parents.slug as network_slug,
-			parents.name as network_name,
-			movies_covers IS NOT NULL as has_cover,
-			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
-			COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
-			row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
-        FROM movies
-		LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
-		LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
-		LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
-        LEFT JOIN entities ON movies.entity_id = entities.id
-        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
-        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
-        LEFT JOIN actors ON local_actors.actor_id = actors.id
-        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
-        LEFT JOIN tags ON local_tags.tag_id = tags.id
-        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
-        LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
-        LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
-        LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
-        ${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
-        GROUP BY
-            movies.id,
-			movies.title,
-			movies.created_at,
-			movies.date,
-			movies_meta.stashed,
-			movies_meta.stashed_scenes,
-			movies_meta.stashed_total,
-			entities.id,
-			entities.name,
-			entities.slug,
-			entities.alias,
-			parents.id,
-			parents.name,
-			parents.slug,
-			parents.alias,
-			movies_covers.*
-    `, movieIds && [movieIds]);
-
-	const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
-
-	const docs = movieIds.map((movieId) => {
-		const movie = moviesById[movieId];
-
-		if (!movie) {
-			return {
-				delete: {
-					index: 'movies',
-					id: movieId,
-				},
-			};
-		}
-
-		const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
-			id: tag.f1,
-			name: tag.f2,
-			priority: tag.f3,
-			alias: tag.f4,
-		}])));
-
-		const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
-		const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
-		const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
-
-		return {
-			replace: {
-				index: 'movies',
-				id: movie.id,
-				doc: {
-					title: movie.title || undefined,
-					title_filtered: filteredTitle || undefined,
-					date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
-					created_at: Math.round(movie.created_at.getTime() / 1000),
-					effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
-					channel_id: movie.channel_id,
-					channel_slug: movie.channel_slug,
-					channel_name: movie.channel_name,
-					network_id: movie.network_id || undefined,
-					network_slug: movie.network_slug || undefined,
-					network_name: movie.network_name || undefined,
-					entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
-					actor_ids: movie.actors.map((actor) => actor.f1),
-					actors: movie.actors.map((actor) => actor.f2).join(),
-					tag_ids: combinedTags.map((tag) => tag.id),
-					tags: flatTags.join(' '),
-					has_cover: movie.has_cover,
-					meta: movie.date ? format(movie.date, 'y yy M MMM MMMM d') : undefined,
-					stashed: movie.stashed || 0,
-					stashed_scenes: movie.stashed_scenes || 0,
-					stashed_total: movie.stashed_total || 0,
-					dupe_index: movie.dupe_index || 0,
-				},
-			},
-		};
-	});
-
-	if (docs.length === 0) {
-		return;
-	}
-
-	await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
-}
-
-async function updateSqlMovieSearch(movieIds, target = 'movie') {
-	logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
-
-	const documents = await knex.raw(`
-		SELECT
-			${target}s.id AS ${target}_id,
-				TO_TSVECTOR(
-				'english',
-					COALESCE(${target}s.title, '') || ' ' ||
-					entities.name || ' ' ||
-					entities.slug || ' ' ||
-					COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
-					COALESCE(parents.name, '') || ' ' ||
-					COALESCE(parents.slug, '') || ' ' ||
-					COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
-					COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
-					STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
-					STRING_AGG(COALESCE(tags.name, ''), ' ')
-			) as document
-		FROM ${target}s
-		LEFT JOIN entities ON ${target}s.entity_id = entities.id
-		LEFT JOIN entities AS parents ON parents.id = entities.parent_id
-		LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
-		LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
-		LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
-		LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
-		LEFT JOIN actors ON actors.id = releases_actors.actor_id
-		LEFT JOIN tags ON tags.id = releases_tags.tag_id
-        ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
-		GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
-    `, movieIds && [movieIds]);
-
-	if (documents.rows?.length > 0) {
-		await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
-	}
+	await syncWeb('scene', releaseIds);
 }
 
 async function updateMovieSearch(releaseIds) {
 	await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;');
 
-	await updateSqlMovieSearch(releaseIds);
-	await updateManticoreMovieSearch(releaseIds);
+	await syncWeb('movie', releaseIds);
+}
+
+async function updateActorSearch(actorIds) {
+	await knex.raw('REFRESH MATERIALIZED VIEW actors_meta;');
+
+	await syncWeb('actor', actorIds);
 }
 
 module.exports = {
 	updateSceneSearch,
 	updateMovieSearch,
+	updateActorSearch,
 };