From 92c2b1866b9bc1f6a11f2118164a24660d0677bc Mon Sep 17 00:00:00 2001
From: DebaucheryLibrarian <DebaucheryLibrarian@unknown.name>
Date: Thu, 29 Feb 2024 01:40:11 +0100
Subject: [PATCH] Sorting aggregated actors by scene count back-end, showing
 disclaimer when limit is reached.

---
 components/filters/actors.vue   | 102 +++++++++++++++++---------------
 components/filters/channels.vue |  54 ++++++++---------
 components/filters/filters.vue  |   9 +++
 components/filters/tags.vue     |  80 ++++++++++++-------------
 config/default.cjs              |   3 +
 renderer/+config.h.js           |   2 +-
 renderer/+onRenderHtml.js       |   3 -
 src/movies.js                   |  22 +++++--
 src/scenes.js                   |  24 ++++++--
 src/web/server.js               |   3 +
 10 files changed, 172 insertions(+), 130 deletions(-)

diff --git a/components/filters/actors.vue b/components/filters/actors.vue
index 1763b01..c4c955c 100644
--- a/components/filters/actors.vue
+++ b/components/filters/actors.vue
@@ -1,56 +1,61 @@
 <template>
 	<div class="filter actors-container">
+		<div
+			v-if="isAggActorsLimited"
+			class="filter-disclaimer"
+		>Some actors may not be listed, apply a filter or search to narrow down results.</div>
+
+		<div class="filters-sort">
+			<input
+				v-model="search"
+				type="search"
+				:placeholder="`Filter ${actors.length} actors`"
+				class="input input-inline filters-search"
+			>
+
+			<div
+				class="filter-sort noselect"
+				@click="selectGender"
+			>
+				<div
+					v-if="!selectedGender"
+					class="gender-unselected"
+				><Icon icon="genders" /></div>
+
+				<Gender
+					v-else
+					:gender="selectedGender"
+					class="gender"
+				/>
+			</div>
+
+			<div
+				v-show="order === 'name'"
+				class="filter-sort order noselect"
+				@click="order = 'count'"
+			>
+				<Icon
+					icon="sort-alpha-asc"
+				/>
+			</div>
+
+			<div
+				v-show="order === 'count'"
+				class="filter-sort order noselect"
+				@click="order = 'name'"
+			>
+				<Icon
+					icon="sort-numeric-desc"
+				/>
+			</div>
+		</div>
+
 		<div
 			v-if="availableActors.length === 0"
 			class="filter-empty"
 		>No actors</div>
 
 		<template v-else>
-			<div class="filters-sort">
-				<input
-					v-model="search"
-					type="search"
-					:placeholder="`Filter ${actors.length} actors`"
-					class="input input-inline filters-search"
-				>
-
-				<div
-					class="filter-sort noselect"
-					@click="selectGender"
-				>
-					<div
-						v-if="!selectedGender"
-						class="gender-unselected"
-					><Icon icon="genders" /></div>
-
-					<Gender
-						v-else
-						:gender="selectedGender"
-						class="gender"
-					/>
-				</div>
-
-				<div
-					v-show="order === 'name'"
-					class="filter-sort order noselect"
-					@click="order = 'count'"
-				>
-					<Icon
-						icon="sort-alpha-asc"
-					/>
-				</div>
-
-				<div
-					v-show="order === 'count'"
-					class="filter-sort order noselect"
-					@click="order = 'name'"
-				>
-					<Icon
-						icon="sort-numeric-desc"
-					/>
-				</div>
-			</div>
-
 			<ul
 				v-for="(actor, index) in selectedActors"
 				:key="`actor-${actor.id}`"
@@ -110,12 +115,14 @@ const emit = defineEmits(['update']);
 const search = ref('');
 const searchRegexp = computed(() => new RegExp(search.value, 'i'));
 const selectedGender = ref(null);
-const order = ref('name');
+const order = ref('count');
 
-const { pageProps } = inject('pageContext');
+const pageContext = inject('pageContext');
+const pageProps = pageContext.pageProps;
 const { actor: pageActor } = pageProps;
 
 const selectedActors = computed(() => props.filters.actors.map((filterActor) => props.actors.find((actor) => actor.id === filterActor.id)).filter(Boolean));
+
 const availableActors = computed(() => props.actors
 	.filter((actor) => !props.filters.actors.some((filterActor) => filterActor.id === actor.id)
 		&& actor.id !== pageActor?.id
@@ -130,6 +137,7 @@ const availableActors = computed(() => props.actors
 	}));
 
 const genders = computed(() => [null, ...['female', 'male', 'transsexual', 'other'].filter((gender) => props.actors.some((actor) => actor.gender === gender))]);
+const isAggActorsLimited = computed(() => props.actors.length >= pageContext.env.maxAggregateSize);
 
 function toggleActor(actor, combine) {
 	if (props.filters.actors.some((filterActor) => filterActor.id === actor.id)) {
diff --git a/components/filters/channels.vue b/components/filters/channels.vue
index b24954b..9366904 100644
--- a/components/filters/channels.vue
+++ b/components/filters/channels.vue
@@ -1,36 +1,36 @@
 <template>
 	<div class="filter channels-container">
+		<div class="filters-sort">
+			<input
+				v-model="search"
+				type="search"
+				:placeholder="`Filter ${channels.length} channels`"
+				class="input input-inline filters-search"
+			>
+
+			<div
+				v-show="order === 'name'"
+				class="filter-sort order noselect"
+				@click="order = 'count'"
+			>
+				<Icon icon="sort-alpha-asc" />
+			</div>
+
+			<div
+				v-show="order === 'count'"
+				class="filter-sort order noselect"
+				@click="order = 'name'"
+			>
+				<Icon icon="sort-numeric-desc" />
+			</div>
+		</div>
+
 		<div
 			v-if="entities.length === 0"
 			class="filter-empty"
 		>No channels</div>
 
 		<template v-else>
-			<div class="filters-sort">
-				<input
-					v-model="search"
-					type="search"
-					:placeholder="`Filter ${channels.length} channels`"
-					class="input input-inline filters-search"
-				>
-
-				<div
-					v-show="order === 'name'"
-					class="filter-sort order noselect"
-					@click="order = 'count'"
-				>
-					<Icon icon="sort-alpha-asc" />
-				</div>
-
-				<div
-					v-show="order === 'count'"
-					class="filter-sort order noselect"
-					@click="order = 'name'"
-				>
-					<Icon icon="sort-numeric-desc" />
-				</div>
-			</div>
-
 			<ul
 				class="filter-items nolist"
 			>
@@ -128,14 +128,14 @@ const entities = computed(() => {
 			return acc;
 		}
 
-		if (!acc[channel.parent.id] && channel.type === 'channel') {
+		if (channel.parent && !acc[channel.parent.id] && channel.type === 'channel') {
 			acc[channel.parent.id] = {
 				...channel.parent,
 				children: [],
 			};
 		}
 
-		if (channel.type === 'channel') {
+		if (channel.parent && channel.type === 'channel') {
 			acc[channel.parent.id].children.push(channel);
 		}
 
diff --git a/components/filters/filters.vue b/components/filters/filters.vue
index 1733284..b2943d3 100644
--- a/components/filters/filters.vue
+++ b/components/filters/filters.vue
@@ -290,6 +290,15 @@ function toggleFilters(state) {
 	color: var(--shadow);
 	font-style: italic;
 }
+
+.filter-disclaimer {
+	background: var(--notice);
+	color: var(--highlight-strong-30);
+	padding: .25rem .5rem;
+	box-shadow: inset 0 0 3px var(--shadow-weak-30);
+	line-height: 1.25;
+	font-size: .9rem;
+}
 </style>
 
 <style scoped>
diff --git a/components/filters/tags.vue b/components/filters/tags.vue
index dcc3967..cae202e 100644
--- a/components/filters/tags.vue
+++ b/components/filters/tags.vue
@@ -1,50 +1,50 @@
 <template>
 	<div class="filter tags-container">
+		<div class="filters-sort">
+			<input
+				v-model="search"
+				type="search"
+				:placeholder="`Filter ${tags.length} tags`"
+				class="input input-inline filters-search"
+			>
+
+			<div
+				v-show="order === 'priority'"
+				class="filter-sort order noselect"
+				@click="order = 'name'"
+			>
+				<Icon
+					icon="star"
+				/>
+			</div>
+
+			<div
+				v-show="order === 'name'"
+				class="filter-sort order noselect"
+				@click="order = 'count'"
+			>
+				<Icon
+					icon="sort-alpha-asc"
+				/>
+			</div>
+
+			<div
+				v-show="order === 'count'"
+				class="filter-sort order noselect"
+				@click="order = 'priority'"
+			>
+				<Icon
+					icon="sort-numeric-desc"
+				/>
+			</div>
+		</div>
+
 		<div
-			v-if="tags.length === 0"
+			v-if="groupedTags.available.length === 0"
 			class="filter-empty"
 		>No tags</div>
 
 		<template v-else>
-			<div class="filters-sort">
-				<input
-					v-model="search"
-					type="search"
-					:placeholder="`Filter ${tags.length} tags`"
-					class="input input-inline filters-search"
-				>
-
-				<div
-					v-show="order === 'priority'"
-					class="filter-sort order noselect"
-					@click="order = 'name'"
-				>
-					<Icon
-						icon="star"
-					/>
-				</div>
-
-				<div
-					v-show="order === 'name'"
-					class="filter-sort order noselect"
-					@click="order = 'count'"
-				>
-					<Icon
-						icon="sort-alpha-asc"
-					/>
-				</div>
-
-				<div
-					v-show="order === 'count'"
-					class="filter-sort order noselect"
-					@click="order = 'priority'"
-				>
-					<Icon
-						icon="sort-numeric-desc"
-					/>
-				</div>
-			</div>
-
 			<ul
 				v-for="(group, groupKey) in groupedTags"
 				:key="groupKey"
diff --git a/config/default.cjs b/config/default.cjs
index 1719630..6fbf76d 100755
--- a/config/default.cjs
+++ b/config/default.cjs
@@ -17,6 +17,9 @@ module.exports = {
 			host: '127.0.0.1',
 			sqlPort: 9306,
 			httpPort: 9308,
+			maxMatches: 2000, // high match count needed primarily for actor aggregations
+			maxAggregateSize: 2000, // must be lower or equal to maxMatches
+			maxQueryTime: 10000,
 		},
 		timeout: 5000,
 		graphiql: false,
diff --git a/renderer/+config.h.js b/renderer/+config.h.js
index fa5e0a2..1ae9309 100644
--- a/renderer/+config.h.js
+++ b/renderer/+config.h.js
@@ -1,3 +1,3 @@
 export default {
-	passToClient: ['pageProps', 'urlPathname', 'routeParams', 'urlParsed'],
+	passToClient: ['pageProps', 'urlPathname', 'routeParams', 'urlParsed', 'env'],
 };
diff --git a/renderer/+onRenderHtml.js b/renderer/+onRenderHtml.js
index e962a6e..e131b5e 100644
--- a/renderer/+onRenderHtml.js
+++ b/renderer/+onRenderHtml.js
@@ -75,9 +75,6 @@ async function onRenderHtml(pageContext) {
 
 	return {
 		documentHtml,
-		pageContext: {
-			// We can add some `pageContext` here, which is useful if we want to do page redirection https://vike.dev/page-redirection
-		},
 	};
 }
 
diff --git a/src/movies.js b/src/movies.js
index ce1d209..a8e3661 100644
--- a/src/movies.js
+++ b/src/movies.js
@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -275,9 +277,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			// sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -285,7 +287,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -294,7 +296,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -318,6 +320,8 @@ export async function fetchMovies(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'movies',
 		query,
@@ -326,8 +330,8 @@ export async function fetchMovies(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
-			max_query_time: 10000,
+			max_matches: config.database.manticore.maxMatches,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -341,16 +345,22 @@ export async function fetchMovies(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const movieIds = result.hits.hits.map((hit) => Number(hit._id));
 	const movies = await fetchMoviesById(movieIds);
 
diff --git a/src/scenes.js b/src/scenes.js
index 12dd726..b7e27b0 100644
--- a/src/scenes.js
+++ b/src/scenes.js
@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -257,9 +259,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -267,7 +269,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -276,7 +278,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -300,6 +302,8 @@ export async function fetchScenes(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'scenes',
 		query,
@@ -308,8 +312,8 @@ export async function fetchScenes(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
-			max_query_time: 10000,
+			max_matches: config.database.manticore.maxMatches,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -323,16 +327,24 @@ export async function fetchScenes(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
+	console.log('hits', result.hits.hits.length);
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
 	const scenes = await fetchScenesById(sceneIds);
 
diff --git a/src/web/server.js b/src/web/server.js
index 66b61fc..ff3cbdc 100644
--- a/src/web/server.js
+++ b/src/web/server.js
@@ -79,6 +79,9 @@ export default async function initServer() {
 		const pageContextInit = {
 			urlOriginal: req.originalUrl,
 			urlQuery: req.query, // vike's own query does not apply boolean parser
+			env: {
+				maxAggregateSize: config.database.manticore.maxAggregateSize,
+			},
 		};
 
 		const pageContext = await renderPage(pageContextInit);