Sorting aggregated actors by scene count back-end, showing disclaimer when limit is reached.

components/filters/actors.vue

@@ -1,11 +1,10 @@
 <template>
 	<div class="filter actors-container">
 		<div
-			v-if="availableActors.length === 0"
+			v-if="isAggActorsLimited"
-			class="filter-empty"
+			class="filter-disclaimer"
-		>No actors</div>
+		>Some actors may not be listed, apply a filter or search to narrow down results.</div>
 
-		<template v-else>
 		<div class="filters-sort">
 			<input
 				v-model="search"
@@ -51,6 +50,12 @@
 			</div>
 		</div>
 
+		<div
+			v-if="availableActors.length === 0"
+			class="filter-empty"
+		>No actors</div>
+
+		<template v-else>
 			<ul
 				v-for="(actor, index) in selectedActors"
 				:key="`actor-${actor.id}`"
@@ -110,12 +115,14 @@ const emit = defineEmits(['update']);
 const search = ref('');
 const searchRegexp = computed(() => new RegExp(search.value, 'i'));
 const selectedGender = ref(null);
-const order = ref('name');
+const order = ref('count');
 
-const { pageProps } = inject('pageContext');
+const pageContext = inject('pageContext');
+const pageProps = pageContext.pageProps;
 const { actor: pageActor } = pageProps;
 
 const selectedActors = computed(() => props.filters.actors.map((filterActor) => props.actors.find((actor) => actor.id === filterActor.id)).filter(Boolean));
+
 const availableActors = computed(() => props.actors
 	.filter((actor) => !props.filters.actors.some((filterActor) => filterActor.id === actor.id)
 		&& actor.id !== pageActor?.id
@@ -130,6 +137,7 @@ const availableActors = computed(() => props.actors
 	}));
 
 const genders = computed(() => [null, ...['female', 'male', 'transsexual', 'other'].filter((gender) => props.actors.some((actor) => actor.gender === gender))]);
+const isAggActorsLimited = computed(() => props.actors.length >= pageContext.env.maxAggregateSize);
 
 function toggleActor(actor, combine) {
 	if (props.filters.actors.some((filterActor) => filterActor.id === actor.id)) {

components/filters/channels.vue

@@ -1,11 +1,5 @@
 <template>
 	<div class="filter channels-container">
-		<div
-			v-if="entities.length === 0"
-			class="filter-empty"
-		>No channels</div>
-
-		<template v-else>
 		<div class="filters-sort">
 			<input
 				v-model="search"
@@ -31,6 +25,12 @@
 			</div>
 		</div>
 
+		<div
+			v-if="entities.length === 0"
+			class="filter-empty"
+		>No channels</div>
+
+		<template v-else>
 			<ul
 				class="filter-items nolist"
 			>
@@ -128,14 +128,14 @@ const entities = computed(() => {
 			return acc;
 		}
 
-		if (!acc[channel.parent.id] && channel.type === 'channel') {
+		if (channel.parent && !acc[channel.parent.id] && channel.type === 'channel') {
 			acc[channel.parent.id] = {
 				...channel.parent,
 				children: [],
 			};
 		}
 
-		if (channel.type === 'channel') {
+		if (channel.parent && channel.type === 'channel') {
 			acc[channel.parent.id].children.push(channel);
 		}
 

components/filters/filters.vue

@@ -290,6 +290,15 @@ function toggleFilters(state) {
 	color: var(--shadow);
 	font-style: italic;
 }
+
+.filter-disclaimer {
+	background: var(--notice);
+	color: var(--highlight-strong-30);
+	padding: .25rem .5rem;
+	box-shadow: inset 0 0 3px var(--shadow-weak-30);
+	line-height: 1.25;
+	font-size: .9rem;
+}
 </style>
 
 <style scoped>

components/filters/tags.vue

@@ -1,11 +1,5 @@
 <template>
 	<div class="filter tags-container">
-		<div
-			v-if="tags.length === 0"
-			class="filter-empty"
-		>No tags</div>
-
-		<template v-else>
 		<div class="filters-sort">
 			<input
 				v-model="search"
@@ -45,6 +39,12 @@
 			</div>
 		</div>
 
+		<div
+			v-if="groupedTags.available.length === 0"
+			class="filter-empty"
+		>No tags</div>
+
+		<template v-else>
 			<ul
 				v-for="(group, groupKey) in groupedTags"
 				:key="groupKey"

config/default.cjs

@@ -17,6 +17,9 @@ module.exports = {
 			host: '127.0.0.1',
 			sqlPort: 9306,
 			httpPort: 9308,
+			maxMatches: 2000, // high match count needed primarily for actor aggregations
+			maxAggregateSize: 2000, // must be lower or equal to maxMatches
+			maxQueryTime: 10000,
 		},
 		timeout: 5000,
 		graphiql: false,

renderer/+config.h.js

@@ -1,3 +1,3 @@
 export default {
-	passToClient: ['pageProps', 'urlPathname', 'routeParams', 'urlParsed'],
+	passToClient: ['pageProps', 'urlPathname', 'routeParams', 'urlParsed', 'env'],
 };

renderer/+onRenderHtml.js

@@ -75,9 +75,6 @@ async function onRenderHtml(pageContext) {
 
 	return {
 		documentHtml,
-		pageContext: {
-			// We can add some `pageContext` here, which is useful if we want to do page redirection https://vike.dev/page-redirection
-		},
 	};
 }
 

src/movies.js

@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -275,9 +277,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			// sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -285,7 +287,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -294,7 +296,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -318,6 +320,8 @@ export async function fetchMovies(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'movies',
 		query,
@@ -326,8 +330,8 @@ export async function fetchMovies(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
+			max_matches: config.database.manticore.maxMatches,
-			max_query_time: 10000,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -341,16 +345,22 @@ export async function fetchMovies(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const movieIds = result.hits.hits.map((hit) => Number(hit._id));
 	const movies = await fetchMoviesById(movieIds);
 

src/scenes.js

@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -257,9 +259,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -267,7 +269,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -276,7 +278,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -300,6 +302,8 @@ export async function fetchScenes(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'scenes',
 		query,
@@ -308,8 +312,8 @@ export async function fetchScenes(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
+			max_matches: config.database.manticore.maxMatches,
-			max_query_time: 10000,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -323,16 +327,24 @@ export async function fetchScenes(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
+	console.log('hits', result.hits.hits.length);
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
 	const scenes = await fetchScenesById(sceneIds);
 

src/web/server.js

@@ -79,6 +79,9 @@ export default async function initServer() {
 		const pageContextInit = {
 			urlOriginal: req.originalUrl,
 			urlQuery: req.query, // vike's own query does not apply boolean parser
+			env: {
+				maxAggregateSize: config.database.manticore.maxAggregateSize,
+			},
 		};
 
 		const pageContext = await renderPage(pageContextInit);