Added filterable stash pages.

src/auth.js

@@ -8,7 +8,9 @@ import { shapes } from '@dicebear/collection';
 import { knexOwner as knex } from './knex.js';
 import { curateUser, fetchUser } from './users.js';
 import { HttpError } from './errors.js';
+import initLogger from './logger.js';
 
+const logger = initLogger();
 const scrypt = util.promisify(crypto.scrypt);
 
 async function verifyPassword(password, storedPassword) {
@@ -32,22 +34,21 @@ async function generateAvatar(user) {
 	});
 
 	await fs.mkdir('media/avatars', { recursive: true });
-
 	await avatar.png().toFile(`media/avatars/${user.id}_${user.username}.png`);
+
+	logger.verbose(`Generated avatar for '${user.username}' (${user.id})`);
 }
 
-export async function login(credentials) {
+export async function login(credentials, userIp) {
 	if (!config.auth.login) {
 		throw new HttpError('Logins are currently disabled', 405);
 	}
 
-	const user = await fetchUser(credentials.username.trim(), {
+	const { user, stashes } = await fetchUser(credentials.username.trim(), {
 		email: true,
 		raw: true,
 	});
 
-	console.log('login user', user);
-
 	if (!user) {
 		throw new HttpError('Username or password incorrect', 401);
 	}
@@ -58,15 +59,21 @@ export async function login(credentials) {
 		.update('last_login', 'NOW()')
 		.where('id', user.id);
 
-	if (!user.avatar) {
+	console.log('login user', user);
+
+	logger.verbose(`Login from '${user.username}' (${user.id}, ${userIp})`);
+
+	try {
+		await fs.access(`media/avatars/${user.id}_${user.username}.png`);
+	} catch (error) {
 		await generateAvatar(user);
 	}
 
 	// fetched the raw user for password verification, don't return directly to user
-	return curateUser(user);
+	return curateUser(user, { stashes });
 }
 
-export async function signup(credentials) {
+export async function signup(credentials, userIp) {
 	if (!config.auth.signup) {
 		throw new HttpError('Sign-ups are currently disabled', 405);
 	}
@@ -126,6 +133,8 @@ export async function signup(credentials) {
 		primary: true,
 	});
 
+	logger.verbose(`Signup from '${curatedUsername}' (${userId}, ${credentials.email}, ${userIp})`);
+
 	await generateAvatar({
 		id: userId,
 		username: curatedUsername,

src/knex.js

@@ -19,4 +19,17 @@ export const knexOwner = knex({
 	// debug: process.env.NODE_ENV === 'development',
 });
 
+export const knexManticore = knex({
+	client: 'mysql',
+	connection: {
+		host: config.database.manticore.host,
+		port: config.database.manticore.sqlPort,
+		database: 'Manticore',
+	},
+	asyncStackTraces: process.env.NODE_ENV === 'development',
+	wrapIdentifier(value, _original, _queryContext) {
+		return value;
+	},
+});
+
 export default knexQuery;

src/manticore.js

@@ -6,3 +6,5 @@ const mantiClient = new manticore.ApiClient();
 mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
 
 export const searchApi = new manticore.SearchApi(mantiClient);
+export const indexApi = new manticore.IndexApi(mantiClient);
+export const utilsApi = new manticore.UtilsApi();

src/scenes.js

@@ -1,7 +1,8 @@
 import config from 'config';
+import util from 'util'; /* eslint-disable-line no-unused-vars */
 
-import { knexOwner as knex } from './knex.js';
-import { searchApi } from './manticore.js';
+import { knexOwner as knex, knexManticore } from './knex.js';
+import { searchApi, utilsApi } from './manticore.js';
 import { HttpError } from './errors.js';
 import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
 import { fetchTagsById } from './tags.js';
@@ -151,6 +152,8 @@ export async function fetchScenesById(sceneIds, reqUser) {
 	}).filter(Boolean);
 }
 
+const sqlImplied = ['scenes_stashed'];
+
 function curateOptions(options) {
 	if (options?.limit > 100) {
 		throw new HttpError('Limit must be <= 100', 400);
@@ -163,10 +166,12 @@ function curateOptions(options) {
 		aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
 		aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
 		aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
+		index: options.index || 'scenes',
+		useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false,
 	};
 }
 
-function buildQuery(filters = {}) {
+function buildQuery(filters = {}, options) {
 	const query = {
 		bool: {
 			must: [],
@@ -210,6 +215,7 @@ function buildQuery(filters = {}) {
 	}
 
 	if (filters.query) {
+		/*
 		query.bool.must.push({
 			bool: {
 				should: [
@@ -224,6 +230,9 @@ function buildQuery(filters = {}) {
 				],
 			},
 		});
+		*/
+
+		query.bool.must.push({ match: { '!title': filters.query } }); // title_filtered is matched instead of title
 	}
 
 	if (filters.tagIds) {
@@ -249,6 +258,10 @@ function buildQuery(filters = {}) {
 		});
 	}
 
+	if (filters.stashId && options.index === 'scenes_stashed') {
+		query.bool.must.push({ equals: { stash_id: filters.stashId } });
+	}
+
 	/* tag filter
 	must_not: [
 		{
@@ -281,6 +294,7 @@ function buildAggregates(options) {
 				field: 'tag_ids',
 				size: config.database.manticore.maxAggregateSize,
 			},
+			sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -290,6 +304,7 @@ function buildAggregates(options) {
 				field: 'channel_id',
 				size: config.database.manticore.maxAggregateSize,
 			},
+			sort: [{ 'count(*)': { order: 'desc' } }],
 		};
 	}
 
@@ -304,20 +319,11 @@ function countAggregations(buckets) {
 	return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
 }
 
-export async function fetchScenes(filters, rawOptions, reqUser) {
-	const options = curateOptions(rawOptions);
-	const { query, sort } = buildQuery(filters);
-
-	console.log('filters', filters);
-	console.log('options', options);
-	console.log('query', query.bool.must);
-
-	console.log('request user', reqUser);
-
-	console.time('manticore');
+async function queryManticoreJson(filters, options, _reqUser) {
+	const { query, sort } = buildQuery(filters, options);
 
 	const result = await searchApi.search({
-		index: 'scenes',
+		index: options.index,
 		query,
 		limit: options.limit,
 		offset: (options.page - 1) * options.limit,
@@ -339,31 +345,181 @@ export async function fetchScenes(filters, rawOptions, reqUser) {
 		},
 	});
 
-	console.timeEnd('manticore');
+	const scenes = result.hits.hits.map((hit) => ({
+		id: hit._id,
+		...hit._source,
+		_score: hit._score,
+	}));
 
-	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets);
-	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets);
-	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets);
+	return {
+		scenes,
+		total: result.hits.total,
+		aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])),
+	};
+}
+
+async function queryManticoreSql(filters, options, _reqUser) {
+	const aggSize = 10 || config.database.manticore.maxAggregateSize;
+
+	const sqlQuery = knexManticore.raw(`
+		:query:
+		OPTION field_weights=(
+			title_filtered=7,
+			 actors=10,
+			 tags=9,
+			 meta=6,
+			 channel_name=2,
+			 channel_slug=3,
+			 network_name=1,
+			 network_slug=1
+		),
+		max_matches=:maxMatches:,
+		max_query_time=:maxQueryTime:
+		:actorsFacet:
+		:tagsFacet:
+		:channelsFacet:
+	`, {
+		query: knexManticore('scenes')
+			.select(knex.raw('*, weight() as _score'))
+			.modify((builder) => {
+				if (filters.stashId) {
+					builder
+						.innerJoin('scenes_stashed', 'scenes.id', 'scenes_stashed.scene_id')
+						.where('scenes_stashed.stash_id', filters.stashId);
+				}
+
+				if (filters.query) {
+					builder.whereRaw('match(\'@!title :query:\', scenes)', { query: filters.query });
+				}
+
+				if (filters.tagIds?.length > 0) {
+					builder.whereIn('any(tag_ids)', filters.tagIds);
+				}
+
+				if (filters.entityId) {
+					builder.where((whereBuilder) => {
+						whereBuilder
+							.where('channel_id', filters.entityId)
+							.orWhere('network_id', filters.entityId);
+					});
+				}
+
+				if (filters.actorIds?.length > 0) {
+					builder.whereIn('any(actor_ids)', filters.actorIds);
+				}
+
+				if (!filters.scope || filters.scope === 'latest') {
+					builder
+						.where('effective_date', '<=', Math.round(Date.now() / 1000))
+						.orderBy('effective_date', 'desc');
+				} else if (filters.scope === 'upcoming') {
+					builder
+						.where('effective_date', '>', Math.round(Date.now() / 1000))
+						.orderBy('effective_date', 'asc');
+				} else if (filters.scope === 'new') {
+					builder.orderBy([
+						{ column: 'created_at', order: 'desc' },
+						{ column: 'effective_date', order: 'asc' },
+					]);
+				} else if (filters.scope === 'likes') {
+					builder.orderBy([
+						{ column: 'stashed', order: 'desc' },
+						{ column: 'effective_date', order: 'desc' },
+					]);
+				} else if (filters.scope === 'results') {
+					builder.orderBy([
+						{ column: '_score', order: 'desc' },
+						{ column: 'effective_date', order: 'desc' },
+					]);
+				} else {
+					builder.orderBy('effective_date', 'desc');
+				}
+			})
+			.limit(options.limit)
+			.toString(),
+		// option threads=1 fixes actors, but drastically slows down performance, wait for fix
+		actorsFacet: options.aggregateActors ? knex.raw('facet actor_ids order by count(*) desc limit ?', [aggSize]) : null,
+		tagsFacet: options.aggregateTags ? knex.raw('facet tag_ids order by count(*) desc limit ?', [aggSize]) : null,
+		channelsFacet: options.aggregateChannels ? knex.raw('facet channel_id order by count(*) desc limit ?', [aggSize]) : null,
+		maxMatches: config.database.manticore.maxMatches,
+		maxQueryTime: config.database.manticore.maxQueryTime,
+	}).toString();
+
+	console.log(sqlQuery);
+
+	const results = await utilsApi.sql(sqlQuery);
+
+	const actorIds = results
+		.find((result) => result.columns[0].actor_ids && result.columns[1]['count(*)'])
+		?.data.map((row) => ({ key: row.actor_ids, doc_count: row['count(*)'] }))
+		|| [];
+
+	const tagIds = results
+		.find((result) => result.columns[0].tag_ids && result.columns[1]['count(*)'])
+		?.data.map((row) => ({ key: row.tag_ids, doc_count: row['count(*)'] }))
+		|| [];
+
+	const channelIds = results
+		.find((result) => result.columns[0].channel_id && result.columns[1]['count(*)'])
+		?.data.map((row) => ({ key: row.channel_id, doc_count: row['count(*)'] }))
+		|| [];
+
+	return {
+		scenes: results[0].data,
+		total: results[0].total,
+		aggregations: {
+			actorIds,
+			tagIds,
+			channelIds,
+		},
+	};
+}
+
+export async function fetchScenes(filters, rawOptions, reqUser) {
+	const options = curateOptions(rawOptions);
+
+	console.log('filters', filters);
+	console.log('options', options);
+
+	/*
+	const result = config.database.manticore.forceSql || filters.stashId
+		? await queryManticoreSql(filters, options, reqUser)
+		: await queryManticoreJson(filters, options, reqUser);
+		*/
+
+	console.time('manticore sql');
+	const result = await queryManticoreSql(filters, options, reqUser);
+	console.timeEnd('manticore sql');
+
+	console.time('manticore json');
+	await queryManticoreJson(filters, options, reqUser);
+	console.timeEnd('manticore json');
+
+	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
+	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
+	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);
 
 	console.time('fetch aggregations');
 
 	const [aggActors, aggTags, aggChannels] = await Promise.all([
-		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
-		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
-		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
+		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [],
+		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [],
+		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [],
 	]);
 
 	console.timeEnd('fetch aggregations');
 
-	const sceneIds = result.hits.hits.map((hit) => Number(hit._id));
+	console.time('fetch full');
+	const sceneIds = result.scenes.map((scene) => Number(scene.id));
 	const scenes = await fetchScenesById(sceneIds, reqUser);
+	console.timeEnd('fetch full');
 
 	return {
 		scenes,
 		aggActors,
 		aggTags,
 		aggChannels,
-		total: result.hits.total,
+		total: result.total,
 		limit: options.limit,
 	};
 }

src/stashes.js

@@ -1,6 +1,7 @@
 import config from 'config';
 
 import { knexOwner as knex } from './knex.js';
+import { indexApi } from './manticore.js';
 import { HttpError } from './errors.js';
 import slugify from './utils/slugify.js';
 import initLogger from './logger.js';
@@ -9,7 +10,7 @@ const logger = initLogger();
 
 let lastActorsViewRefresh = 0;
 
-export function curateStash(stash) {
+export function curateStash(stash, assets = {}) {
 	if (!stash) {
 		return null;
 	}
@@ -24,6 +25,12 @@ export function curateStash(stash) {
 		stashedScenes: stash.stashed_scenes || null,
 		stashedMovies: stash.stashed_movies || null,
 		stashedActors: stash.stashed_actors || null,
+		user: assets.user ? {
+			id: assets.user.id,
+			username: assets.user.username,
+			avatar: `/media/avatars/${assets.user.id}_${assets.user.username}.png`,
+			createdAt: assets.user.created_at,
+		} : null,
 	};
 
 	return curatedStash;
@@ -40,23 +47,39 @@ function curateStashEntry(stash, user) {
 	return curatedStashEntry;
 }
 
-export async function fetchStash(stashId, sessionUser) {
-	if (!sessionUser) {
-		throw new HttpError('You are not authenthicated', 401);
+function verifyStashAccess(stash, sessionUser) {
+	if (!stash || (!stash.public && stash.user_id !== sessionUser?.id)) {
+		throw new HttpError('This stash does not exist, or you are not allowed access.', 404);
+	}
+}
+
+export async function fetchStashById(stashId, sessionUser) {
+	const stash = await knex('stashes')
+		.where('id', stashId)
+		.first();
+
+	verifyStashAccess(stash, sessionUser);
+
+	return curateStash(stash);
+}
+
+export async function fetchStashByUsernameAndSlug(username, stashSlug, sessionUser) {
+	const user = await knex('users').where('username', username).first();
+
+	if (!user) {
+		throw new HttpError('This user does not exist.', 404);
 	}
 
 	const stash = await knex('stashes')
-		.where({
-			id: stashId,
-			user_id: sessionUser.id,
-		})
+		.select('stashes.*', 'stashes_meta.*')
+		.leftJoin('stashes_meta', 'stashes_meta.stash_id', 'stashes.id')
+		.where('slug', stashSlug)
+		.where('user_id', user.id)
 		.first();
 
-	if (!stash) {
-		throw new HttpError('You are not authorized to access this stash', 403);
-	}
+	verifyStashAccess(stash, sessionUser);
 
-	return curateStash(stash);
+	return curateStash(stash, { user });
 }
 
 export async function fetchStashes(domain, itemId, sessionUser) {
@@ -145,7 +168,7 @@ export async function refreshActorsView() {
 }
 
 export async function stashActor(actorId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
+	const stash = await fetchStashById(stashId, sessionUser);
 
 	await knex('stashes_actors')
 		.insert({
@@ -158,30 +181,6 @@ export async function stashActor(actorId, stashId, sessionUser) {
 	return fetchStashes('actor', actorId, sessionUser);
 }
 
-export async function stashScene(sceneId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
-
-	await knex('stashes_scenes')
-		.insert({
-			stash_id: stash.id,
-			scene_id: sceneId,
-		});
-
-	return fetchStashes('scene', sceneId, sessionUser);
-}
-
-export async function stashMovie(movieId, stashId, sessionUser) {
-	const stash = await fetchStash(stashId, sessionUser);
-
-	await knex('stashes_movies')
-		.insert({
-			stash_id: stash.id,
-			movie_id: movieId,
-		});
-
-	return fetchStashes('movie', movieId, sessionUser);
-}
-
 export async function unstashActor(actorId, stashId, sessionUser) {
 	await knex
 		.from('stashes_actors AS deletable')
@@ -198,6 +197,18 @@ export async function unstashActor(actorId, stashId, sessionUser) {
 	return fetchStashes('actor', actorId, sessionUser);
 }
 
+export async function stashScene(sceneId, stashId, sessionUser) {
+	const stash = await fetchStashById(stashId, sessionUser);
+
+	await knex('stashes_scenes')
+		.insert({
+			stash_id: stash.id,
+			scene_id: sceneId,
+		});
+
+	return fetchStashes('scene', sceneId, sessionUser);
+}
+
 export async function unstashScene(sceneId, stashId, sessionUser) {
 	await knex
 		.from('stashes_scenes AS deletable')
@@ -209,9 +220,34 @@ export async function unstashScene(sceneId, stashId, sessionUser) {
 			.where('stashes.user_id', sessionUser.id))
 		.delete();
 
+	await indexApi.callDelete({
+		index: 'scenes_stashed',
+		query: {
+			bool: {
+				must: [
+					{ equals: { id: sceneId } },
+					{ equals: { stash_id: stashId } },
+					{ equals: { user_id: sessionUser.id } },
+				],
+			},
+		},
+	});
+
 	return fetchStashes('scene', sceneId, sessionUser);
 }
 
+export async function stashMovie(movieId, stashId, sessionUser) {
+	const stash = await fetchStashById(stashId, sessionUser);
+
+	await knex('stashes_movies')
+		.insert({
+			stash_id: stash.id,
+			movie_id: movieId,
+		});
+
+	return fetchStashes('movie', movieId, sessionUser);
+}
+
 export async function unstashMovie(movieId, stashId, sessionUser) {
 	await knex
 		.from('stashes_movies AS deletable')

src/tools/manticore-joins.js

@@ -0,0 +1,82 @@
+import { indexApi, utilsApi } from '../manticore.js';
+import rawMovies from './movies.json' with { type: 'json' };
+
+async function fetchMovies() {
+	const movies = rawMovies
+		.filter((movie) => movie.cast.length > 0
+			&& movie.genres.length > 0
+			&& movie.cast.every((actor) => actor.charCodeAt(0) >= 65)) // throw out movies with non-alphanumerical actor names
+		.map((movie, index) => ({ id: index, ...movie }));
+
+	const actors = Array.from(new Set(movies.flatMap((movie) => movie.cast))).sort();
+	const genres = Array.from(new Set(movies.flatMap((movie) => movie.genres)));
+
+	return {
+		movies,
+		actors,
+		genres,
+	};
+}
+
+async function init() {
+	await utilsApi.sql('drop table if exists movies');
+	await utilsApi.sql('drop table if exists movies_liked');
+
+	await utilsApi.sql(`create table movies (
+		 id int,
+		 title text,
+		 actor_ids multi,
+		 actors text,
+		 genre_ids multi,
+		 genres text
+	)`);
+
+	await utilsApi.sql(`create table movies_liked (
+		 id int,
+		 user_id int,
+		 movie_id int
+	)`);
+
+	const { movies, actors, genres } = await fetchMovies();
+
+	const likedMovieIds = Array.from(new Set(Array.from({ length: 10.000 }, () => movies[Math.round(Math.random() * movies.length)].id)));
+
+	const docs = movies
+		.map((movie) => ({
+			replace: {
+				index: 'movies',
+				id: movie.id,
+				doc: {
+					title: movie.title,
+					actor_ids: movie.cast.map((actor) => actors.indexOf(actor)),
+					actors: movie.cast.join(','),
+					genre_ids: movie.genres.map((genre) => genres.indexOf(genre)),
+					genres: movie.genres.join(','),
+				},
+			},
+		}))
+		.concat(likedMovieIds.map((movieId, index) => ({
+			replace: {
+				index: 'movies_liked',
+				id: index + 1,
+				doc: {
+					user_id: Math.floor(Math.random() * 51),
+					movie_id: movieId,
+				},
+			},
+		})));
+
+	const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
+
+	console.log('data', data);
+
+	const result = await utilsApi.sql(`
+		select * from movies_liked
+		limit 10
+	`);
+
+	console.log(result[0].data);
+	console.log(result[1]);
+}
+
+init();

src/tools/manticore-scenes.js

@@ -0,0 +1,203 @@
+// import config from 'config';
+import { format } from 'date-fns';
+import { faker } from '@faker-js/faker';
+
+import { indexApi, utilsApi } from '../manticore.js';
+
+import { knexOwner as knex } from '../knex.js';
+import slugify from '../utils/slugify.js';
+import chunk from '../utils/chunk.js';
+
+async function fetchScenes() {
+	const scenes = await knex.raw(`
+        SELECT
+            releases.id AS id,
+			releases.title,
+			releases.created_at,
+			releases.date,
+			releases.shoot_id,
+			scenes_meta.stashed,
+			entities.id as channel_id,
+			entities.slug as channel_slug,
+			entities.name as channel_name,
+			parents.id as network_id,
+			parents.slug as network_slug,
+			parents.name as network_name,
+			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
+			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
+        FROM releases
+		LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
+        LEFT JOIN entities ON releases.entity_id = entities.id
+        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
+        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
+        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
+        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
+        LEFT JOIN actors ON local_actors.actor_id = actors.id
+        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
+        LEFT JOIN tags ON local_tags.tag_id = tags.id
+        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
+        GROUP BY
+            releases.id,
+			releases.title,
+			releases.created_at,
+			releases.date,
+			releases.shoot_id,
+			scenes_meta.stashed,
+			entities.id,
+			entities.name,
+			entities.slug,
+			entities.alias,
+			parents.id,
+			parents.name,
+			parents.slug,
+			parents.alias;
+	`);
+
+	const actors = Object.fromEntries(scenes.rows.flatMap((row) => row.actors.map((actor) => [actor.f1, faker.person.fullName()])));
+	const tags = Object.fromEntries(scenes.rows.flatMap((row) => row.tags.map((tag) => [tag.f1, faker.word.adjective()])));
+
+	return scenes.rows.map((row) => {
+		const title = faker.lorem.lines(1);
+
+		const channelName = faker.company.name();
+		const channelSlug = slugify(channelName, '');
+
+		const networkName = faker.company.name();
+		const networkSlug = slugify(networkName, '');
+
+		const rowActors = row.actors.map((actor) => ({ f1: actor.f1, f2: actors[actor.f1] }));
+		const rowTags = row.tags.map((tag) => ({ f1: tag.f1, f2: tags[tag.f1], f3: tag.f3 }));
+
+		return {
+			...row,
+			title,
+			actors: rowActors,
+			tags: rowTags,
+			channel_name: channelName,
+			channel_slug: channelSlug,
+			network_name: networkName,
+			network_slug: networkSlug,
+		};
+	});
+}
+
+async function updateStashed(docs) {
+	await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
+		await chain;
+
+		const sceneIds = docsChunk.map((doc) => doc.replace.id);
+
+		const stashes = await knex('stashes_scenes')
+			.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes.id as stash_id', 'stashes.user_id as user_id')
+			.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
+			.whereIn('scene_id', sceneIds);
+
+		if (stashes.length > 0) {
+			console.log(stashes);
+		}
+
+		const stashDocs = docsChunk.flatMap((doc) => {
+			const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
+
+			if (sceneStashes.length === 0) {
+				return [];
+			}
+
+			const stashDoc = sceneStashes.map((stash) => ({
+				replace: {
+					index: 'movies_liked',
+					id: stash.stashed_id,
+					doc: {
+						// ...doc.replace.doc,
+						movie_id: doc.replace.id,
+						user_id: stash.user_id,
+					},
+				},
+			}));
+
+			return stashDoc;
+		});
+
+		console.log(stashDocs);
+
+		if (stashDocs.length > 0) {
+			await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
+		}
+	}, Promise.resolve());
+}
+
+async function init() {
+	await utilsApi.sql('drop table if exists movies');
+	await utilsApi.sql('drop table if exists movies_liked');
+
+	await utilsApi.sql(`create table movies (
+		 id int,
+		 title text,
+		 title_filtered text,
+		 channel_id int,
+		 channel_name text,
+		 channel_slug text,
+		 network_id int,
+		 network_name text,
+		 network_slug text,
+		 actor_ids multi,
+		 actors text,
+		 tag_ids multi,
+		 tags text,
+		 meta text,
+		 date timestamp,
+		 created_at timestamp,
+		 effective_date timestamp,
+		 liked int
+	)`);
+
+	await utilsApi.sql(`create table movies_liked (
+		movie_id int,
+		user_id int
+	)`);
+
+	const scenes = await fetchScenes();
+
+	const docs = scenes.map((scene) => {
+		const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
+		const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
+		const filteredTitle = scene.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'i'), ''), scene.title).trim().replace(/\s{2,}/, ' ');
+
+		return {
+			replace: {
+				index: 'movies',
+				id: scene.id,
+				doc: {
+					title: scene.title || undefined,
+					title_filtered: filteredTitle || undefined,
+					date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
+					created_at: Math.round(scene.created_at.getTime() / 1000),
+					effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
+					// shoot_id: scene.shoot_id || undefined,
+					channel_id: scene.channel_id,
+					channel_slug: scene.channel_slug,
+					channel_name: scene.channel_name,
+					network_id: scene.network_id || undefined,
+					network_slug: scene.network_slug || undefined,
+					network_name: scene.network_name || undefined,
+					actor_ids: scene.actors.map((actor) => actor.f1),
+					actors: scene.actors.map((actor) => actor.f2).join(),
+					tag_ids: scene.tags.map((tag) => tag.f1),
+					tags: flatTags.join(' '),
+					meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
+					liked: scene.stashed || 0,
+				},
+			},
+		};
+	});
+
+	const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
+
+	await updateStashed(docs);
+
+	console.log('data', data);
+
+	knex.destroy();
+}
+
+init();

src/users.js

@@ -46,17 +46,17 @@ export async function fetchUser(userId, options = {}) {
 		.groupBy('users.id', 'users_roles.role')
 		.first();
 
+	const stashes = await knex('stashes')
+		.where('user_id', user.id)
+		.leftJoin('stashes_meta', 'stashes_meta.stash_id', 'stashes.id');
+
 	if (!user) {
 		throw HttpError(`User '${userId}' not found`, 404);
 	}
 
 	if (options.raw) {
-		return user;
+		return { user, stashes };
 	}
 
-	const stashes = await knex('stashes')
-		.where('user_id', user.id)
-		.leftJoin('stashes_meta', 'stashes_meta.stash_id', 'stashes.id');
-
 	return curateUser(user, { stashes });
 }

src/users/curate.js

@@ -0,0 +1,21 @@
+export function curateUser(user, assets = {}) {
+	if (!user) {
+		return null;
+	}
+
+	const curatedStashes = assets.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [];
+
+	const curatedUser = {
+		id: user.id,
+		username: user.username,
+		email: user.email,
+		emailVerified: user.email_verified,
+		identityVerified: user.identity_verified,
+		avatar: `/media/avatars/${user.id}_${user.username}.png`,
+		createdAt: user.created_at,
+		stashes: curatedStashes,
+		primaryStash: curatedStashes.find((stash) => stash.primary),
+	};
+
+	return curatedUser;
+}

src/utils/chunk.js

@@ -0,0 +1,4 @@
+export default function chunk(array, chunkSize = 1000) {
+	return Array.from({ length: Math.ceil(array.length / chunkSize) })
+		.map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
+}

src/web/auth.js

@@ -1,16 +1,42 @@
 /* eslint-disable no-param-reassign */
+import IPCIDR from 'ip-cidr';
+
 import { login, signup } from '../auth.js';
 
+function getIp(req) {
+	const ip = req.headers['x-forwarded-for']?.split(',')[0] || req.connection.remoteAddress; // See src/ws
+
+	const unmappedIp = ip?.includes('.')
+		? ip.slice(ip.lastIndexOf(':') + 1)
+		: ip;
+
+	// ensure IP is in expanded notation for consistency and matching
+	const expandedIp = unmappedIp.includes(':')
+		? new IPCIDR(`${ip}/128`) // IPv6
+		: new IPCIDR(`${ip}/32`); // IPv4
+
+	if (!expandedIp.addressStart?.addressMinusSuffix) {
+		throw new Error(`Could not determine user IP from ${ip}`);
+	}
+
+	return expandedIp.addressStart?.addressMinusSuffix || null;
+}
+
 export async function setUserApi(req, res, next) {
+	const ip = getIp(req);
+
+	req.userIp = ip;
+
 	if (req.session.user) {
 		req.user = req.session.user;
+		req.user.ip = ip;
 	}
 
 	next();
 }
 
 export async function loginApi(req, res) {
-	const user = await login(req.body);
+	const user = await login(req.body, req.userIp);
 
 	req.session.user = user;
 	res.send(user);
@@ -27,7 +53,7 @@ export async function logoutApi(req, res) {
 }
 
 export async function signupApi(req, res) {
-	const user = await signup(req.body);
+	const user = await signup(req.body, req.userIp);
 
 	req.session.user = user;
 	res.send(user);

src/web/scenes.js

@@ -11,6 +11,7 @@ export async function curateScenesQuery(query) {
 		actorIds: [query.actorId, ...(query.actors?.split(',') || []).map((identifier) => parseActorIdentifier(identifier)?.id)].filter(Boolean),
 		tagIds: await getIdsBySlug([query.tagSlug, ...(query.tags?.split(',') || [])], 'tags'),
 		entityId: query.e ? await getIdsBySlug([query.e], 'entities').then(([id]) => id) : query.entityId,
+		stashId: Number(query.stashId),
 	};
 }
 

src/web/server.js

@@ -135,7 +135,14 @@ export default async function initServer() {
 		const pageContextInit = {
 			urlOriginal: req.originalUrl,
 			urlQuery: req.query, // vike's own query does not apply boolean parser
-			user: req.user,
+			user: req.user && {
+				id: req.user.id,
+				username: req.user.username,
+				email: req.user.email,
+				avatar: req.user.avatar,
+				stashes: req.user.stashes,
+				primaryStash: req.user.primaryStash,
+			},
 			env: {
 				maxAggregateSize: config.database.manticore.maxAggregateSize,
 			},

src/web/stashes.js

@@ -53,7 +53,7 @@ export async function unstashActorApi(req, res) {
 }
 
 export async function unstashSceneApi(req, res) {
-	const stashes = await unstashScene(req.params.sceneId, req.params.stashId, req.user);
+	const stashes = await unstashScene(Number(req.params.sceneId), Number(req.params.stashId), req.user);
 
 	res.send(stashes);
 }