import process from 'node:process';
import config from 'config';
import { MerkleJson } from 'merkle-json';

import escape from '../utils/escape-manticore.js';
import promiseProps from '../utils/promise-props.js';
import { curateActor, fetchActorsById, sortActorsByGender } from './actors.js';
import { getAffiliateSceneUrl } from './affiliates.js';
import argv from './argv.js';
import { censor } from './censor.js';
import { fetchEntitiesById } from './entities.js';
import { HttpError } from './errors.js';
import { knexQuery as knex, knexManticore, knexOwner } from './knex.js';
import initLogger from './logger.js';
import { querySql } from './manticore.js';
import { curateMedia } from './media.js';
import { fetchMoviesById } from './movies.js';
import { curateRevision } from './revisions.js';
import { curateStash } from './stashes.js';
import { syncScenes } from './sync.js';
import { fetchTagsById } from './tags.js';

const logger = initLogger();
const mj = new MerkleJson();

function curateScene(rawScene, assets, reqUser, context) {
	if (!rawScene) {
		return null;
	}

	const curatedScene = {
		id: rawScene.id,
		title: censor(rawScene.title, context.restriction),
		slug: rawScene.slug,
		url: rawScene.url,
		entryId: rawScene.entry_id,
		date: rawScene.date,
		datePrecision: rawScene.date_precision,
		isUpcoming: rawScene.date && rawScene.date - new Date() > 0,
		createdAt: rawScene.created_at,
		effectiveDate: rawScene.effective_date,
		description: censor(rawScene.description, context.restriction),
		duration: rawScene.duration,
		shootId: rawScene.shoot_id,
		productionDate: rawScene.production_date,
		channel: {
			id: assets.channel.id,
			slug: assets.channel.slug,
			name: censor(assets.channel.name, context.restriction),
			type: assets.channel.type,
			url: assets.channel.url,
			isIndependent: assets.channel.independent,
			hasLogo: assets.channel.has_logo,
		},
		network: assets.channel.network_id
			? {
					id: assets.channel.network_id,
					slug: assets.channel.network_slug,
					name: censor(assets.channel.network_name, context.restriction),
					url: assets.network_url,
					type: assets.channel.network_type,
					hasLogo: assets.channel.network_has_logo,
				}
			: null,
		studio: assets.studio
			? {
					id: assets.studio.id,
					slug: assets.studio.slug,
					name: assets.studio.name,
					type: assets.studio.type,
					hasLogo: assets.studio.has_logo,
				}
			: null,
		affiliate: assets.channel.affiliate
			? {
					id: assets.channel.affiliate.id,
					url: assets.channel.affiliate.url,
					parameters: assets.channel.affiliate.parameters || {},
					entityId: assets.channel.affiliate.entity_id,
				}
			: null,
		actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, {
			sceneDate: rawScene.effective_date,
			stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id),
		})), { title: rawScene.title }),
		directors: assets.directors.map((director) => ({
			id: director.id,
			slug: director.slug,
			name: director.name,
		})),
		tags: assets.tags.map((tag) => ({
			id: tag.id,
			slug: tag.slug,
			name: censor(tag.name, context.restriction),
			priority: tag.priority,
			actorId: tag.actor_id,
		})),
		chapters: assets.chapters.map((chapter) => ({
			id: chapter.id,
			title: chapter.title,
			description: chapter.description,
			time: chapter.time,
			date: chapter.date,
			duration: chapter.duration,
			poster: context.restriction
				? null
				: curateMedia(chapter.chapter_poster, { type: 'poster' }),
			tags: chapter.chapter_tags.map((tag) => ({
				id: tag.id,
				name: tag.name,
				slug: tag.slug,
			})),
		})),
		qualities: rawScene.qualities?.sort((qualityA, qualityB) => qualityB - qualityA) || [],
		photoCount: rawScene.photo_count,
		movies: assets.movies.map((movie) => ({
			id: movie.id,
			slug: movie.slug,
			title: censor(movie.title, context.restriction),
			covers: movie.movie_covers && !context.restriction
				? movie.movie_covers.map((cover) => curateMedia(cover, { type: 'cover' })).toSorted((coverA, coverB) => coverA.index - coverB.index)
				: [],
		})),
		series: assets.series.map((serie) => ({
			id: serie.id,
			slug: serie.slug,
			title: serie.title,
			poster: context.restriction
				? null
				: (serie.serie_poster, { type: 'poster' }),
		})),
		poster: curateMedia(assets.poster, { type: 'poster' }),
		photos: assets.photos?.map((photo) => curateMedia(photo, { type: 'photo' })) || [],
		caps: assets.caps?.map((cap) => curateMedia(cap, { type: 'cap' })) || [],
		fingerprints: assets.fingerprints?.map((fingerprint) => ({
			hash: fingerprint.hash,
			type: fingerprint.type,
			duration: fingerprint.duration,
			source: fingerprint.source,
			submissions: fingerprint.source_submissions,
			createdAt: fingerprint.created_at,
		})) || [],
		createdBatchId: rawScene.created_batch_id,
		updatedBatchId: rawScene.updated_batch_id,
		isNew: assets.lastBatchId === rawScene.created_batch_id,
		stashes: assets.stashes?.map((stash) => curateStash(stash)) || [],
		alerts: {
			only: assets.alerts?.filter((alert) => alert.is_only).flatMap((alert) => alert.alert_ids) || [],
			multi: assets.alerts?.filter((alert) => !alert.is_only).flatMap((alert) => alert.alert_ids) || [],
		},
	};

	const isVideoRestricted = config.media.videoRestrictions.includes(curatedScene.channel.slug) || config.media.videoRestrictions.includes(`_${curatedScene.network?.slug}`);

	if (!isVideoRestricted || reqUser?.abilities?.some((ability) => ability.trailerAccess)) {
		if (reqUser) {
			// only show trailers to logged in users to curb S3 traffic
			curatedScene.trailer = curateMedia(assets.trailer, { type: 'trailer', isRestricted: isVideoRestricted });
		}

		curatedScene.teaser = curateMedia(assets.teaser, { type: 'teaser', isRestricted: isVideoRestricted });
	}

	curatedScene.watchUrl = getAffiliateSceneUrl(curatedScene);

	return curatedScene;
}

export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
	const {
		scenes,
		channels,
		studios,
		actors,
		directors,
		tags,
		chapters,
		movies,
		series,
		posters,
		photos,
		caps,
		trailers,
		teasers,
		fingerprints,
		stashes,
		alerts,
		lastBatch,
	} = await promiseProps({
		scenes: knex('releases').whereIn('releases.id', sceneIds),
		channels: knex('releases')
			.select(
				'channels.*',
				'networks.id as network_id',
				'networks.slug as network_slug',
				'networks.name as network_name',
				'networks.type as network_type',
				'networks.url as network_url',
				'networks.has_logo as network_has_logo',
				knex.raw('row_to_json(affiliates) as affiliate'),
			)
			.whereIn('releases.id', sceneIds)
			.leftJoin('entities as channels', 'channels.id', 'releases.entity_id')
			.leftJoin('entities as networks', 'networks.id', 'channels.parent_id')
			// .leftJoin('affiliates as channel_affiliates', 'channel_affiliates.entity_id', 'channels.id')
			// .leftJoin('affiliates as network_affiliates', 'network_affiliates.entity_id', 'networks.id')
			.joinRaw(`
				left join lateral (
					select *
					from affiliates
					where affiliates.entity_id in (channels.id, networks.id)
					order by (affiliates.entity_id = channels.id) desc
					limit 1
				) affiliates ON TRUE
			`),
		studios: knex('releases')
			.select('studios.*')
			.whereIn('releases.id', sceneIds)
			.innerJoin('entities as studios', 'studios.id', 'releases.studio_id'),
		actors: knex('releases_actors')
			.select(
				'actors.*',
				knex.raw('row_to_json(avatars) as avatar'),
				knex.raw('row_to_json(sfw_media) as sfw_avatar'),
				knex.raw('row_to_json(media_biometrics) as biometrics'),
				knex.raw('row_to_json(aliases) as alias'),
				knex.raw('case when aliases.id is not null then json_build_object(\'id\', aliases.id, \'name\', aliases.name, \'slug\', aliases.slug) end as alias'),
				'countries.name as birth_country_name',
				'countries.alias as birth_country_alias',
				'releases_actors.release_id',
			)
			.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
			.leftJoin('actors as aliases', 'aliases.id', 'releases_actors.alias_id')
			.leftJoin('media as avatars', 'avatars.id', 'actors.avatar_media_id')
			.leftJoin('media as sfw_media', 'sfw_media.id', 'avatars.sfw_media_id')
			.leftJoin('media_biometrics', 'media_biometrics.media_id', 'actors.avatar_media_id')
			.leftJoin('countries', 'countries.alpha2', 'actors.birth_country_alpha2')
			.whereIn('release_id', sceneIds)
			.groupBy('actors.id', 'aliases.id', 'releases_actors.release_id', 'avatars.id', 'media_biometrics.id', 'countries.name', 'countries.alias', 'sfw_media.id'),
		directors: knex('releases_directors')
			.whereIn('release_id', sceneIds)
			.leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'),
		tags: knex('releases_tags')
			.select('tags.id', 'tags.slug', 'tags.name', 'tags.priority', 'releases_tags.release_id', 'actor_id')
			.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
			.whereNotNull('tags.id')
			.whereIn('release_id', sceneIds)
			.orderBy('priority', 'desc'),
		chapters: knex('chapters')
			.select(
				'chapters.*',
				knex.raw('coalesce(json_agg(tags) filter (where tags.id is not null), \'[]\') as chapter_tags'),
				knex.raw('row_to_json(posters) as chapter_poster'),
			)
			.leftJoin('chapters_tags', 'chapters_tags.chapter_id', 'chapters.id')
			.leftJoin('tags', 'tags.id', 'chapters_tags.tag_id')
			.leftJoin('chapters_posters', 'chapters_posters.chapter_id', 'chapters.id')
			.leftJoin('media as posters', 'posters.id', 'chapters_posters.media_id')
			.whereIn('chapters.release_id', sceneIds)
			.groupBy('chapters.id', 'posters.id')
			.orderBy('time', 'asc'),
		movies: context.includePartOf
			? knex('movies_scenes')
					.select('movies_scenes.scene_id', 'movies.*', knex.raw('json_agg(media) as movie_covers'))
					.leftJoin('movies', 'movies.id', 'movies_scenes.movie_id')
					.leftJoin('movies_covers', 'movies_covers.movie_id', 'movies.id')
					.leftJoin('media', 'media.id', 'movies_covers.media_id')
					.whereIn('scene_id', sceneIds)
					.groupBy('movies.id', 'movies_scenes.scene_id')
			: [],
		series: context.includePartOf
			? knex('series_scenes')
					.select('series_scenes.scene_id', 'series.*', knex.raw('row_to_json(media) as serie_poster'))
					.leftJoin('series', 'series.id', 'series_scenes.serie_id')
					.leftJoin('series_posters', 'series_posters.serie_id', 'series.id')
					.leftJoin('media', 'media.id', 'series_posters.media_id')
					.whereIn('scene_id', sceneIds)
					.groupBy('series.id', 'series_scenes.scene_id', 'media.*')
			: [],
		posters: knex('releases_posters')
			.select('media.*', 'releases_posters.release_id', knex.raw('row_to_json(sfw_media) as sfw_media'))
			.whereIn('release_id', sceneIds)
			.leftJoin('media', 'media.id', 'releases_posters.media_id')
			.leftJoin('media as sfw_media', 'sfw_media.id', 'media.sfw_media_id')
			.groupBy('media.id', 'releases_posters.release_id', 'sfw_media.id'),
		photos: context.includeAssets && !context.restriction
			? knex.transaction(async (trx) => {
					if (reqUser) {
						await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
					}

					return trx('releases_photos')
						.select('media.*', 'releases_photos.release_id', knex.raw('row_to_json(sfw_media) as sfw_media'))
						.leftJoin('media', 'media.id', 'releases_photos.media_id')
						.leftJoin('media as sfw_media', 'sfw_media.id', 'media.sfw_media_id')
						.whereIn('release_id', sceneIds)
						.orderBy('index')
						.groupBy('media.id', 'releases_photos.release_id', 'sfw_media.id');
				})
			: [],
		caps: context.includeAssets
			? knex.transaction(async (trx) => {
					if (reqUser) {
						await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
					}

					return trx('releases_caps')
						.select('media.*', 'releases_caps.release_id', knex.raw('row_to_json(sfw_media) as sfw_media'))
						.leftJoin('media', 'media.id', 'releases_caps.media_id')
						.leftJoin('media as sfw_media', 'sfw_media.id', 'media.sfw_media_id')
						.whereIn('release_id', sceneIds)
						.orderBy('index')
						.groupBy('media.id', 'releases_caps.release_id', 'sfw_media.id');
				})
			: [],
		trailers: context.includeAssets
			? knex.transaction(async (trx) => {
					if (reqUser) {
						await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
					}

					return trx('releases_trailers')
						.whereIn('release_id', sceneIds)
						.leftJoin('media', 'media.id', 'releases_trailers.media_id');
				})
			: [],
		teasers: context.includeAssets
			? knex.transaction(async (trx) => {
					if (reqUser) {
						await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
					}

					return trx('releases_teasers')
						.whereIn('release_id', sceneIds)
						.leftJoin('media', 'media.id', 'releases_teasers.media_id');
				})
			: [],
		fingerprints: context.includeAssets
			? knex.transaction(async (trx) => {
					if (reqUser) {
						await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
					}

					return trx('releases_fingerprints')
						.select('scene_id', 'hash', 'type', 'duration', 'source', 'source_submissions', knex.raw('min(coalesce(source_created_at, created_at)) as created_at'))
						.whereIn('scene_id', sceneIds)
						.orderBy([
							{ column: 'source_submissions', order: 'desc' },
							{ column: knex.raw('min(coalesce(source_created_at, created_at))'), order: 'desc' },
						])
						.groupBy(['scene_id', 'hash', 'type', 'duration', 'source', 'source_submissions']);
				})
			: [],
		lastBatch: knex('batches')
			.select('id')
			.where('showcased', true)
			.orderBy('created_at', 'desc')
			.first(),
		stashes: reqUser
			? knexOwner('stashes_scenes')
					.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
					.where('stashes.user_id', reqUser.id)
					.whereIn('stashes_scenes.scene_id', sceneIds)
			: [],
		alerts: reqUser
			? knex('alerts_users_scenes')
					.where('user_id', reqUser.id)
					.whereIn('scene_id', sceneIds)
			: [],
	});

	const actorStashes = reqUser && context.actorStashes
		? await knexOwner('stashes_actors')
				.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
				.where('stashes.user_id', reqUser.id)
				.whereIn('stashes_actors.actor_id', actors.map((actor) => actor.id))
		: [];

	return sceneIds.map((sceneId) => {
		const scene = scenes.find((sceneEntry) => sceneEntry.id === sceneId);

		if (!scene) {
			return null;
		}

		const sceneChannel = channels.find((entity) => entity.id === scene.entity_id);
		const sceneStudio = studios.find((entity) => entity.id === scene.studio_id);
		const sceneActors = actors.filter((actor) => actor.release_id === sceneId);
		const sceneDirectors = directors.filter((director) => director.release_id === sceneId);
		const sceneTags = tags.filter((tag) => tag.release_id === sceneId);
		const sceneChapters = chapters.filter((chapter) => chapter.release_id === sceneId);
		const sceneMovies = movies.filter((movie) => movie.scene_id === sceneId);
		const sceneSeries = series.filter((serie) => serie.scene_id === sceneId);
		const scenePoster = posters.find((poster) => poster.release_id === sceneId);
		const scenePhotos = photos.filter((photo) => photo.release_id === sceneId);
		const sceneCaps = caps.filter((cap) => cap.release_id === sceneId);
		const sceneTrailers = trailers.find((trailer) => trailer.release_id === sceneId);
		const sceneTeasers = teasers.find((teaser) => teaser.release_id === sceneId);
		const sceneFingerprints = fingerprints.filter((fingerprint) => fingerprint.scene_id === sceneId);
		const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId);
		const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean);
		const sceneAlerts = alerts.filter((alert) => alert.scene_id === sceneId);

		return curateScene(scene, {
			channel: sceneChannel,
			studio: sceneStudio,
			actors: sceneActors,
			directors: sceneDirectors,
			tags: sceneTags,
			chapters: sceneChapters,
			movies: sceneMovies,
			series: sceneSeries,
			poster: scenePoster,
			photos: scenePhotos,
			caps: sceneCaps,
			trailer: sceneTrailers,
			teaser: sceneTeasers,
			fingerprints: sceneFingerprints,
			stashes: sceneStashes,
			actorStashes: sceneActorStashes,
			alerts: sceneAlerts,
			lastBatchId: lastBatch?.id,
		}, reqUser, context);
	}).filter(Boolean);
}

const sqlImplied = ['scenes_stashed'];

function curateOptions(options) {
	if (options?.limit > 100) {
		throw new HttpError('Limit must be <= 100', 400);
	}

	return {
		limit: options?.limit || 30,
		page: Number(options?.page) || 1,
		aggregate: options?.aggregate ?? true,
		aggregateYears: (options?.aggregate ?? true) && (options?.aggregateYears ?? true),
		aggregateActors: (options?.aggregate ?? true) && (options?.aggregateActors ?? true),
		aggregateTags: (options?.aggregate ?? true) && (options?.aggregateTags ?? true),
		aggregateChannels: (options?.aggregate ?? true) && (options?.aggregateChannels ?? true),
		dedupe: !!options?.dedupe,
		index: options?.index || 'scenes',
		useSql: options?.useSql || (typeof options?.useSql === 'undefined' && sqlImplied.includes(options?.index)) || false,
	};
}

// function curateFacet(results, field, count = 'count(distinct id)') {
function curateFacet(results, field) {
	return results
		.find((result) => result.columns[0][field] && (result.columns[1]['count(distinct id)'] || result.columns[1]['count(*)']))
		?.data
		.map((row) => ({ key: row[field], doc_count: row['count(distinct id)'] || row['count(*)'] }))
		.filter((row) => !!row.key)
		|| [];
}

const packN = 100_000;

function mergePackedTags(tags) {
	const mergedCounts = tags.reduce((merged, tag) => {
		const tagId = tag.key % packN;

		merged.set(tagId, (merged.get(tagId) ?? 0) + tag.doc_count);

		return merged;
	}, new Map());

	return Array.from(mergedCounts.entries(), ([key, count]) => ({ key, doc_count: count }));
}

async function queryManticoreSql(filters, options, _reqUser) {
	const aggSize = config.database.manticore.maxAggregateSize;

	const sqlQuery = knexManticore.raw(`
		:query:
		OPTION
			field_weights=(
				title_filtered=7,
				actors=10,
				tags=9,
				movies=7,
				meta=6,
				channel_name=2,
				channel_slug=3,
				network_name=1,
				network_slug=1
			),
			max_matches=:maxMatches:,
			max_query_time=:maxQueryTime:
		:yearsFacet:
		:actorsFacet:
		:tagsFacet:
		:channelsFacet:
		:studiosFacet:;
		show meta;
	`, {
		query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes')
			.modify((builder) => {
				if (filters.stashId) {
					builder.select(knex.raw(`
						scenes.id as id,
						scenes.title as title,
						scenes.actor_ids as actor_ids,
						scenes.entity_ids as entity_ids,
						scenes.tag_ids as tag_ids,
						scenes.channel_id as channel_id,
						scenes.network_id as network_id,
						scenes.effective_date as effective_date,
						scenes.stashed as stashed,
						scenes.created_at,
						created_at as stashed_at,
						year(scenes.effective_date) as effective_year,
						weight() as _score
					`));

					builder
						.innerJoin('scenes', 'scenes.id', 'scenes_stashed.scene_id')
						.where('stash_id', filters.stashId);
				}
				else {
					builder.select(knex.raw(`
						*,
						year(scenes.effective_date) as effective_year,
						weight() as _score
					`));
				}

				if (filters.query) {
					// we exclude title because we have a curated title_filtered field for more effective results
					builder.whereRaw('match(\'@!title :query:\', scenes)', { query: escape(filters.query) });
				}

				if (filters.years?.length > 0) {
					builder.whereIn('effective_year', filters.years);
				}

				filters.tagIds?.forEach((tagId) => {
					if (filters.onlyActorTags) {
						builder.where((whereBuilder) => {
							whereBuilder.where('any(assigned_tag_ids)', tagId);

							filters.actorIds?.forEach((actorId) => {
								whereBuilder.orWhere('any(assigned_tag_ids)', actorId * 1_000_00 + tagId);
							});
						});
					}
					else {
						builder.where('any(tag_ids)', tagId);
					}
				});

				if (filters.notTagIds) {
					builder.whereNotIn('tag_ids', filters.notTagIds);
				}

				filters.actorIds?.forEach((actorId) => {
					builder.where('any(actor_ids)', actorId);
				});

				if (filters.notActorIds) {
					builder.whereNotIn('actor_ids', filters.notActorIds);
				}

				if (filters.entityId) {
					builder.whereRaw('any(entity_ids) = ?', filters.entityId);

					/* manticore does not currently support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068
					builder.where((whereBuilder) => {
						whereBuilder
							.where('scenes.channel_id', filters.entityId)
							.orWhere('scenes.network_id', filters.entityId);
					});
					*/
				}

				if (filters.notEntityIds) {
					builder.whereNotIn('entity_ids', filters.notEntityIds);
				}

				if (filters.movieId) {
					builder.whereRaw('any(movie_ids) = ?', filters.movieId);
				}

				if (filters.serieId) {
					builder.whereRaw('any(serie_ids) = ?', filters.serieId);
				}

				if (typeof filters.isShowcased === 'boolean') {
					builder.where('scenes.is_showcased', filters.isShowcased);
				}

				if (options.dedupe) {
					builder.where('scenes.dupe_index', '<', 2);
				}

				if (!filters.scope || filters.scope === 'latest') {
					builder
						.where('effective_date', '<=', Math.round(Date.now() / 1000))
						.orderBy('scenes.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works
				}
				else if (filters.scope === 'upcoming') {
					builder
						.where('effective_date', '>', Math.round(Date.now() / 1000))
						.orderBy('scenes.effective_date', 'asc');
				}
				else if (filters.scope === 'new') {
					builder.orderBy([
						{ column: 'scenes.created_at', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'asc' },
					]);
				}
				else if (filters.scope === 'likes') {
					builder.orderBy([
						{ column: 'scenes.stashed', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				}
				else if (filters.scope === 'results') {
					builder.orderBy([
						{ column: '_score', order: 'desc' },
						{ column: 'scenes.stashed', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				}
				else if (filters.scope === 'stashed' && filters.stashId) {
					builder.orderBy([
						{ column: 'stashed_at', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				}
				else if (filters.scope === 'oldest') {
					builder.orderBy('scenes.effective_date', 'asc');
				}
				else {
					builder.orderBy('scenes.effective_date', 'desc');
				}
			})
			.limit(options.limit)
			.offset((options.page - 1) * options.limit),
		// option threads=1 fixes actors, but drastically slows down performance, wait for fix
		yearsFacet: options.aggregateYears ? knex.raw('facet effective_year as years_facet order by effective_year desc limit ?', [aggSize]) : null,
		actorsFacet: options.aggregateActors ? knex.raw('facet scenes.actor_ids as actors_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
		tagsFacet: options.aggregateTags ? knex.raw('facet scenes.assigned_tag_ids as tags_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
		channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id as channels_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
		studiosFacet: options.aggregateChannels ? knex.raw('facet scenes.studio_id as studios_facet distinct id order by count(*) desc limit ?', [aggSize]) : null,
		maxMatches: config.database.manticore.maxMatches,
		maxQueryTime: config.database.manticore.maxQueryTime,
	}).toString();

	// manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around
	const curatedSqlQuery = filters.stashId
		? sqlQuery
		: sqlQuery
				.replace(/scenes\./g, '')
				.replace(/scenes_\./g, 'scenes.');

	if (process.env.NODE_ENV === 'development' && argv.debug) {
		console.log(curatedSqlQuery);
	}

	const results = await querySql(curatedSqlQuery);

	const years = curateFacet(results, 'years_facet');
	const actorIds = curateFacet(results, 'actors_facet');
	const tagIds = curateFacet(results, 'tags_facet');
	const channelIds = curateFacet(results, 'channels_facet');
	const studioIds = curateFacet(results, 'studios_facet');

	const allTagIds = mergePackedTags(tagIds);

	const actorTagIds = mergePackedTags(tagIds.filter((tag) => {
		if (tag.key < packN || !filters?.actorIds.length) {
			// global
			return true;
		}

		const tagActorId = Math.floor(tag.key / packN);

		if (filters.actorIds.includes(tagActorId)) {
			return true;
		}

		return false;
	}));

	const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;

	return {
		scenes: results[0].data,
		total,
		aggregations: {
			years,
			actorIds,
			tagIds: allTagIds,
			actorTagIds,
			channelIds,
			studioIds,
		},
	};
}

function countAggregations(buckets) {
	if (!buckets) {
		return null;
	}

	return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}

export async function fetchScenes(filters, rawOptions, reqUser, context) {
	const options = curateOptions(rawOptions);

	if (argv.debug) {
		console.log('filters', filters);
		console.log('options', options);
	}

	console.time('manticore sql');

	const result = await queryManticoreSql(filters, options, reqUser);

	console.timeEnd('manticore sql');

	const aggYears = options.aggregateYears && result.aggregations.years.map((bucket) => ({ year: bucket.key, count: bucket.doc_count }));
	const entityIds = options.aggregateChannels && [...(result.aggregations.channelIds || []), ...(result.aggregations.studioIds || [])];

	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
	const channelCounts = options.aggregateChannels && countAggregations(entityIds);

	console.time('fetch aggregations');

	const [aggActors, aggTags, aggActorTags, aggChannels] = await Promise.all([
		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { shallow: true, order: ['slug', 'asc'], append: actorCounts }, reqUser) : [],
		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(tags.name)'), 'asc'], append: tagCounts }, reqUser, context) : [],
		options.aggregateTags ? fetchTagsById(result.aggregations.actorTagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(tags.name)'), 'asc'], append: tagCounts }, reqUser, context) : [],
		options.aggregateChannels ? fetchEntitiesById(entityIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: channelCounts }, reqUser, context) : [],
	]);

	console.timeEnd('fetch aggregations');

	console.time('fetch full');
	const sceneIds = result.scenes.map((scene) => Number(scene.id));
	const scenes = await fetchScenesById(sceneIds, { reqUser, ...context });
	console.timeEnd('fetch full');

	return {
		scenes,
		aggYears,
		aggActors,
		aggTags,
		aggActorTags,
		aggChannels,
		total: result.total,
		limit: options.limit,
	};
}

export async function fetchSceneRevisions(revisionId, filters = {}, reqUser) {
	const limit = filters.limit || 50;
	const page = filters.page || 1;

	const revisions = await knexOwner('scenes_revisions')
		.select(
			'scenes_revisions.*',
			'users.username as username',
			'reviewers.username as reviewer_username',
		)
		.leftJoin('users', 'users.id', 'scenes_revisions.user_id')
		.leftJoin('users as reviewers', 'reviewers.id', 'scenes_revisions.reviewed_by')
		.modify((builder) => {
			if (!['admin', 'editor'].includes(reqUser?.role) && !filters.userId && !filters.sceneId) {
				builder.where('user_id', reqUser.id);
			}

			if (filters.userId) {
				if (!['admin', 'editor'].includes(reqUser?.role) && filters.userId !== reqUser.id) {
					throw new HttpError('You are not permitted to view revisions from other users.', 403);
				}

				builder.where('scenes_revisions.user_id', filters.userId);
			}

			if (revisionId) {
				builder.where('scenes_revisions.id', revisionId);
				return;
			}

			if (filters.sceneId) {
				builder.where('scenes_revisions.scene_id', filters.sceneId);
			}

			if (filters.isFinalized === false) {
				builder.whereNull('approved');
			}

			if (filters.isFinalized === true) {
				builder.whereNotNull('approved');
			}
		})
		.orderBy('created_at', 'desc')
		.limit(limit)
		.offset((page - 1) * limit);

	const actorIds = Array.from(new Set(revisions.flatMap((revision) => [
		...revision.base.actors,
		...(revision.deltas.find((delta) => delta.key === 'actors')?.value || []),
		...revision.base.tags.map((tag) => tag.actorId),
		...revision.deltas.find((delta) => delta.key === 'tags')?.value.map((tag) => tag.actorId) || [],
	].filter(Boolean))));

	const tagIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.tags, ...(revision.deltas.find((delta) => delta.key === 'tags')?.value || [])].map((tag) => tag.id))));
	const movieIds = Array.from(new Set(revisions.flatMap((revision) => [...revision.base.movies, ...(revision.deltas.find((delta) => delta.key === 'movies')?.value || [])])));

	const [actors, tags, movies] = await Promise.all([
		fetchActorsById(actorIds),
		fetchTagsById(tagIds),
		fetchMoviesById(movieIds),
	]);

	const curatedRevisions = revisions.map((revision) => curateRevision(revision));

	return {
		revisions: curatedRevisions,
		revision: revisionId && curatedRevisions.find((revision) => revision.id === revisionId),
		actors,
		tags,
		movies,
	};
}

const keyMap = {
	datePrecision: 'date_precision',
	productionDate: 'production_date',
	productionLocation: 'production_location',
	productionCity: 'production_city',
	productionState: 'production_state',
};

async function applySceneValueDelta(sceneId, delta, trx) {
	return knexOwner('releases')
		.where('id', sceneId)
		.update(keyMap[delta.key] || delta.key, delta.value)
		.transacting(trx);
}

async function applySceneActorsDelta(sceneId, delta, trx) {
	await knexOwner('releases_actors')
		.where('release_id', sceneId)
		.delete()
		.transacting(trx);

	if (delta.value.length > 0) {
		await knexOwner('releases_actors')
			.insert(delta.value.map((actorId) => ({
				release_id: sceneId,
				actor_id: actorId,
			})))
			.transacting(trx);
	}
}

async function applySceneTagsDelta(sceneId, delta, trx) {
	// don't remove unidentified tags
	await knexOwner('releases_tags')
		.where('release_id', sceneId)
		.whereNotNull('tag_id')
		.delete()
		.transacting(trx);

	if (delta.value.length > 0) {
		await knexOwner('releases_tags')
			.insert(delta.value.map((tag) => ({
				release_id: sceneId,
				tag_id: tag.id,
				actor_id: tag.actorId,
				source: 'editor',
			})))
			.transacting(trx);
	}
}

async function applySceneMoviesDelta(sceneId, delta, trx) {
	await knexOwner('movies_scenes')
		.where('scene_id', sceneId)
		.delete()
		.transacting(trx);

	if (delta.value.length > 0) {
		await knexOwner('movies_scenes')
			.insert(delta.value.map((movieId) => ({
				scene_id: sceneId,
				movie_id: movieId,
			})))
			.transacting(trx);
	}
}

async function applySceneDeleteDelta(sceneId, _delta, trx, reqUser) {
	if (!reqUser.abilities.some((ability) => ability.subject === 'scene' && ability.action === 'delete')) {
		throw new HttpError('You are not privileged to delete scenes', 400);
	}

	await knexOwner('releases')
		.where('id', sceneId)
		.delete()
		.transacting(trx);
}

async function applySceneRevision(revisionIds, reqUser) {
	const revisions = await knexOwner('scenes_revisions')
		.whereIn('id', revisionIds)
		.whereNull('applied_at'); // should not re-apply revision that was already applied

	await revisions.reduce(async (chain, revision) => {
		await chain;

		await knexOwner.transaction(async (trx) => {
			await Promise.all(revision.deltas.map(async (delta) => {
				if (delta.key === 'delete') {
					return applySceneDeleteDelta(revision.scene_id, delta, trx, reqUser);
				}

				if ([
					'title',
					'description',
					'date',
					'url',
					'datePrecision',
					'duration',
					'productionDate',
					'productionLocation',
					'productionCity',
					'productionState',
				].includes(delta.key)) {
					return applySceneValueDelta(revision.scene_id, delta, trx);
				}

				if (delta.key === 'actors') {
					return applySceneActorsDelta(revision.scene_id, delta, trx);
				}

				if (delta.key === 'tags') {
					return applySceneTagsDelta(revision.scene_id, delta, trx);
				}

				if (delta.key === 'movies') {
					return applySceneMoviesDelta(revision.scene_id, delta, trx);
				}

				return null;
			}));

			await knexOwner('scenes_revisions')
				.where('id', revision.id)
				.update('applied_at', knexOwner.fn.now())
				.transacting(trx);

			// await trx.commit();
		}).catch(async (error) => {
			logger.error(`Failed to apply revision ${revision.id} on scene ${revision.scene_id}: ${error.message}`);
			throw error;
		});
	}, Promise.resolve());

	const sceneIds = Array.from(new Set(revisions.map((revision) => revision.scene_id)));

	await syncScenes(sceneIds);
}

export async function reviewSceneRevision(revisionId, isApproved, { feedback }, reqUser) {
	if (!reqUser || reqUser.role === 'user') {
		throw new HttpError('You are not permitted to approve revisions', 403);
	}

	if (typeof isApproved !== 'boolean') {
		throw new HttpError('You must either approve or reject the revision', 400);
	}

	const updated = await knexOwner('scenes_revisions')
		.where('id', revisionId)
		.whereNull('approved') // don't rerun reviewed revision, must be forked into new revision instead
		.whereNull('applied_at')
		.update({
			approved: isApproved,
			reviewed_at: knex.fn.now(),
			reviewed_by: reqUser.id,
			feedback,
		});

	if (updated === 0) {
		throw new HttpError('This revision was already reviewed', 409);
	}

	if (isApproved) {
		try {
			await applySceneRevision([revisionId], reqUser);
		}
		catch (error) {
			await knexOwner('scenes_revisions')
				.where('id', revisionId)
				.update({
					approved: null,
					reviewed_at: null,
					reviewed_by: null,
				});

			throw error;
		}
	}
}

export async function createSceneRevision(sceneId, { edits, comment, apply }, reqUser) {
	if (!reqUser) {
		throw new HttpError('Must be authenticated to create scene revision', 401);
	}

	const [
		[scene],
		openRevisions,
	] = await Promise.all([
		fetchScenesById([sceneId], {
			reqUser,
			includeAssets: true,
			includePartOf: true,
		}),
		knexOwner('scenes_revisions')
			.where('user_id', reqUser.id)
			.whereNull('approved'),
	]);

	if (!scene) {
		throw new HttpError(`No scene with ID ${sceneId} found to update`, 404);
	}

	if (openRevisions.length >= config.revisions.unapprovedLimit && reqUser.role !== 'admin') {
		throw new HttpError(`You have ${config.revisions.unapprovedLimit} unapproved revisions, please wait for approval before submitting another revision.`, 429);
	}

	const baseScene = Object.fromEntries(Object.entries(scene).map(([key, values]) => {
		if ([
			'effectiveDate',
			'isNew',
			'network',
			'stashes',
			'watchUrl',
		].includes(key)) {
			return null;
		}

		if (values?.hash) {
			return [key, values.hash];
		}

		if (values?.id) {
			return [key, values.id];
		}

		if (key === 'tags') {
			return [key, values.map((tag) => ({
				id: tag.id,
				actorId: tag.actorId,
			}))];
		}

		if (Array.isArray(values)) {
			return [key, values.map((value) => value?.hash || value?.id || value)];
		}

		return [key, values];
	}).filter(Boolean));

	const deltas = Object.entries(edits).map(([key, value]) => {
		if (key === 'delete') {
			return { key: 'delete' };
		}

		if (baseScene[key] === value || typeof value === 'undefined') {
			return null;
		}

		if (key === 'tags') {
			return {
				key,
				value: value.map((tag) => ({
					id: tag.id,
					actorId: tag.actorId,
				})),
			};
		}

		if (Array.isArray(value)) {
			const valueSet = new Set(value);
			const baseSet = new Set(baseScene[key]);

			if (valueSet.size === baseSet.size && baseScene[key].every((id) => valueSet.has(id))) {
				return null;
			}

			return { key, value: Array.from(valueSet) };
		}

		return { key, value };
	}).filter(Boolean);

	if (deltas.length === 0) {
		throw new HttpError('No effective changes provided', 400);
	}

	const [revisionEntry] = await knexOwner('scenes_revisions')
		.insert({
			user_id: reqUser.id,
			scene_id: scene.id,
			base: JSON.stringify(baseScene),
			deltas: JSON.stringify(deltas),
			hash: mj.hash({
				base: baseScene,
				deltas,
			}),
			comment,
		})
		.returning('id');

	if (['admin', 'editor'].includes(reqUser.role) && apply) {
		// don't keep the editor waiting for the revision to apply
		reviewSceneRevision(revisionEntry.id, true, {}, reqUser).catch(() => {});
	}
}
