Using packed keys instead of actor-tags table.

pages/tags/@tagId/+Page.vue

@@ -1,7 +1,10 @@
 <template>
 	<div class="page">
 		<div class="header">
-			<h2 class="title">{{ tag.name }}</h2>
+			<h2
+				:title="`${tag.name} (#${tag.id})`"
+				class="title"
+			>{{ tag.name }}</h2>
 
 			<Heart
 				domain="tags"

src/scenes.js

@@ -418,6 +418,20 @@ function curateFacet(results, field) {
 		|| [];
 }
 
+const packN = 100_000;
+
+function mergePackedTags(tags) {
+	const mergedCounts = tags.reduce((merged, tag) => {
+		const tagId = tag.key % packN;
+
+		merged.set(tagId, (merged.get(tagId) ?? 0) + tag.doc_count);
+
+		return merged;
+	}, new Map());
+
+	return Array.from(mergedCounts.entries(), ([key, count]) => ({ key, doc_count: count }));
+}
+
 async function queryManticoreSql(filters, options, _reqUser) {
 	const aggSize = config.database.manticore.maxAggregateSize;
 
@@ -440,7 +454,6 @@ async function queryManticoreSql(filters, options, _reqUser) {
 		:yearsFacet:
 		:actorsFacet:
 		:tagsFacet:
-		:actorTagsFacet:
 		:channelsFacet:
 		:studiosFacet:;
 		show meta;
@@ -473,11 +486,6 @@ async function queryManticoreSql(filters, options, _reqUser) {
 						year(scenes.effective_date) as effective_year,
 						weight() as _score
 					`));
-
-					// manticore only supports one joined table, so we can't use it inside stashes
-					builder
-						.leftJoin('scenes_tags', 'scenes_tags.scene_id', 'scenes_.id')
-						.groupBy('scenes.id');
 				}
 
 				if (filters.query) {
@@ -533,12 +541,6 @@ async function queryManticoreSql(filters, options, _reqUser) {
 					builder.where('scenes.is_showcased', filters.isShowcased);
 				}
 
-				/*
-				if (filters.isShowcased) {
-					builder.where('scenes.date', '>', 0);
-				}
-				*/
-
 				if (options.dedupe) {
 					builder.where('scenes.dupe_index', '<', 2);
 				}
@@ -583,11 +585,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
 		// option threads=1 fixes actors, but drastically slows down performance, wait for fix
 		yearsFacet: options.aggregateYears ? knex.raw('facet effective_year as years_facet order by effective_year desc limit ?', [aggSize]) : null,
 		actorsFacet: options.aggregateActors ? knex.raw('facet scenes.actor_ids as actors_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
-		// don't facet tags associated to other actors, actor ID 0 means global
-		tagsFacet: options.aggregateTags ? knex.raw('facet scenes.tag_ids as tags_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
-		actorTagsFacet: options.aggregateTags && !filters.stashId // eslint-disable-line no-nested-ternary
-			? knex.raw(`facet IF(IN(scenes_tags.actor_id, ${[0, ...filters?.actorIds || []]}), scenes_tags.tag_id, 0) as actor_tags_facet distinct id order by count(*) desc limit ?`, [aggSize])
-			: null,
+		tagsFacet: options.aggregateTags ? knex.raw('facet scenes.assigned_tag_ids as tags_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
 		channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id as channels_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
 		studiosFacet: options.aggregateChannels ? knex.raw('facet scenes.studio_id as studios_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
 		maxMatches: config.database.manticore.maxMatches,
@@ -610,10 +608,26 @@ async function queryManticoreSql(filters, options, _reqUser) {
 	const years = curateFacet(results, 'years_facet');
 	const actorIds = curateFacet(results, 'actors_facet');
 	const tagIds = curateFacet(results, 'tags_facet');
-	const actorTagIds = curateFacet(results, 'actor_tags_facet');
 	const channelIds = curateFacet(results, 'channels_facet');
 	const studioIds = curateFacet(results, 'studios_facet');
 
+	const allTagIds = mergePackedTags(tagIds);
+
+	const actorTagIds = mergePackedTags(tagIds.filter((tag) => {
+		if (tag.key < packN || !filters?.actorIds.length) {
+			// global
+			return true;
+		}
+
+		const tagActorId = Math.floor(tag.key / packN);
+
+		if (filters.actorIds.includes(tagActorId)) {
+			return true;
+		}
+
+		return false;
+	}));
+
 	const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;
 
 	return {
@@ -622,7 +636,7 @@ async function queryManticoreSql(filters, options, _reqUser) {
 		aggregations: {
 			years,
 			actorIds,
-			tagIds,
+			tagIds: allTagIds,
 			actorTagIds,
 			channelIds,
 			studioIds,

src/sync.js

@@ -115,7 +115,7 @@ export async function syncManticoreScenes(sceneIds) {
 			grandparents.id as parent_network_id,
 			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
 			COALESCE(JSON_AGG(DISTINCT (actors_aliases.id, actors_aliases.name)) FILTER (WHERE actors_aliases.id IS NOT NULL), '[]') as actors_aliases,
-			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
+			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
 			COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
 			COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
 			studios.showcased IS NOT false
@@ -187,7 +187,17 @@ export async function syncManticoreScenes(sceneIds) {
 		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
 		const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
 
-		// TODO: reconsider how direct vs indirect tags are stored and searched
+		// use decimal packing with 5-decimal pad to allow for actor-specific tags, i.e. actor 135 tag 5 = 13500005
+		// all global tags are necessarily < 10,000, all tags for actor 135 are >= 13500000 and <= 13599999
+		// f1 = tag ID, f5 = actor ID
+		const assignedTagIds = scene.tags.map((tag) => (tag.f5 === null ? tag.f1 : tag.f5 * 1_000_00 + tag.f1));
+
+		/*
+		if (sceneId === '187734') {
+			console.log(scene, assignedTagIds);
+			throw new Error('ABORT');
+		}
+		*/
 
 		return {
 			replace: {
@@ -213,7 +223,8 @@ export async function syncManticoreScenes(sceneIds) {
 					entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
 					actor_ids: scene.actors.map((actor) => actor.f1), // don't include aliases in ID or they would show up in filters
 					actors: Array.from(new Set([...scene.actors.map((actor) => actor.f2), ...scene.actors_aliases.map((actor) => actor.f2)])).join(),
-					tag_ids: scene.tags.map((tag) => tag.f1),
+					tag_ids: Array.from(new Set(scene.tags.map((tag) => tag.f1))),
+					assigned_tag_ids: assignedTagIds,
 					tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
 					movie_ids: scene.movies.map((movie) => movie.f1),
 					movies: scene.movies.map((movie) => movie.f2).join(' '),

tools/manticore-scenes.js

@@ -25,6 +25,7 @@ async function init() {
 			 actor_ids multi,
 			 actors text,
 			 tag_ids multi,
+			 assigned_tag_ids multi64,
 			 tags text,
 			 movie_ids multi,
 			 movies text,
@@ -41,12 +42,15 @@ async function init() {
 		)`);
 
 		await utilsApi.sql('drop table if exists scenes_tags');
+
+		/* legacy, using packed decimal keys now
 		await utilsApi.sql(`create table scenes_tags (
 			 id int,
 			 scene_id int,
 			 tag_id int,
 			 actor_id int
 		)`);
+		*/
 
 		console.log('Recreated scenes tables, syncing scenes...');