Sorting aggregated actors by scene count back-end, showing disclaimer when limit is reached.

src/movies.js

@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -275,9 +277,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			// sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -285,7 +287,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -294,7 +296,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -318,6 +320,8 @@ export async function fetchMovies(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'movies',
 		query,
@@ -326,8 +330,8 @@ export async function fetchMovies(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
-			max_query_time: 10000,
+			max_matches: config.database.manticore.maxMatches,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -341,16 +345,22 @@ export async function fetchMovies(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const movieIds = result.hits.hits.map((hit) => Number(hit._id));
 	const movies = await fetchMoviesById(movieIds);
 

src/scenes.js

@@ -1,3 +1,5 @@
+import config from 'config';
+
 import knex from './knex.js';
 import { searchApi } from './manticore.js';
 import { HttpError } from './errors.js';
@@ -257,9 +259,9 @@ function buildAggregates(options) {
 		aggregates.actorIds = {
 			terms: {
 				field: 'actor_ids',
-				size: 5000,
+				size: config.database.manticore.maxAggregateSize,
 			},
-			// sort: [{ doc_count: { order: 'asc' } }],
+			sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -267,7 +269,7 @@ function buildAggregates(options) {
 		aggregates.tagIds = {
 			terms: {
 				field: 'tag_ids',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -276,7 +278,7 @@ function buildAggregates(options) {
 		aggregates.channelIds = {
 			terms: {
 				field: 'channel_id',
-				size: 1000,
+				size: config.database.manticore.maxAggregateSize,
 			},
 		};
 	}
@@ -300,6 +302,8 @@ export async function fetchScenes(filters, rawOptions) {
 	console.log('options', options);
 	console.log('query', query.bool.must);
 
+	console.time('manticore');
+
 	const result = await searchApi.search({
 		index: 'scenes',
 		query,
@@ -308,8 +312,8 @@ export async function fetchScenes(filters, rawOptions) {
 		sort,
 		aggs: buildAggregates(options),
 		options: {
-			max_matches: 1000,
-			max_query_time: 10000,
+			max_matches: config.database.manticore.maxMatches,
+			max_query_time: config.database.manticore.maxQueryTime,
 			field_weights: {
 				title_filtered: 7,
 				actors: 10,
@@ -323,16 +327,24 @@ export async function fetchScenes(filters, rawOptions) {
 		},
 	});
 
+	console.timeEnd('manticore');
+
+	console.log('hits', result.hits.hits.length);
+
 	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
 	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
 	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
 
+	console.time('fetch aggregations');
+
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
 		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
 		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
 		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
+	console.timeEnd('fetch aggregations');
+
 	const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
 	const scenes = await fetchScenesById(sceneIds);
 

src/web/server.js

@@ -79,6 +79,9 @@ export default async function initServer() {
 		const pageContextInit = {
 			urlOriginal: req.originalUrl,
 			urlQuery: req.query, // vike's own query does not apply boolean parser
+			env: {
+				maxAggregateSize: config.database.manticore.maxAggregateSize,
+			},
 		};
 
 		const pageContext = await renderPage(pageContextInit);