traxxx-web/src/scenes.js

import config from 'config';
import util from 'util'; /* eslint-disable-line no-unused-vars */

import { knexQuery as knex, knexOwner, knexManticore } from './knex.js';
import { utilsApi } from './manticore.js';
import { HttpError } from './errors.js';
import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js';
import { fetchTagsById } from './tags.js';
import { fetchEntitiesById } from './entities.js';
import { curateStash } from './stashes.js';
import { curateMedia } from './media.js';
import escape from '../utils/escape-manticore.js';
import promiseProps from '../utils/promise-props.js';

function getWatchUrl(scene) {
	if (scene.url) {
		return scene.url;
	}

	if (scene.channel && (scene.channel.isIndependent || scene.channel.type === 'network')) {
		return scene.channel.url;
	}

	if (scene.network) {
		return scene.network.url;
	}

	return null;
}

function getAffiliateUrl(scene) {
	const watchUrl = getWatchUrl(scene);

	if (!watchUrl) {
		return null;
	}

	if (!scene.affiliate?.parameters) {
		return scene.url;
	}

	const newParams = new URLSearchParams({
		...Object.fromEntries(new URL(watchUrl).searchParams),
		...Object.fromEntries(new URLSearchParams(scene.affiliate.parameters)),
	});

	return `${watchUrl}?${newParams.toString()}`;
}

function curateScene(rawScene, assets) {
	if (!rawScene) {
		return null;
	}

	const curatedScene = {
		id: rawScene.id,
		title: rawScene.title,
		slug: rawScene.slug,
		url: rawScene.url,
		date: rawScene.date,
		createdAt: rawScene.created_at,
		effectiveDate: rawScene.effective_date,
		description: rawScene.description,
		duration: rawScene.duration,
		shootId: rawScene.shoot_id,
		channel: {
			id: assets.channel.id,
			slug: assets.channel.slug,
			name: assets.channel.name,
			type: assets.channel.type,
			isIndependent: assets.channel.independent,
			hasLogo: assets.channel.has_logo,
		},
		network: assets.channel.network_id ? {
			id: assets.channel.network_id,
			slug: assets.channel.network_slug,
			name: assets.channel.network_name,
			type: assets.channel.network_type,
			hasLogo: assets.channel.has_logo,
		} : null,
		affiliate: assets.channel.affiliate ? {
			id: assets.channel.affiliate.id,
			url: assets.channel.affiliate.url,
			parameters: assets.channel.affiliate.parameters,
		} : null,
		actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, {
			sceneDate: rawScene.effective_date,
			stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id),
		})), { title: rawScene.title }),
		directors: assets.directors.map((director) => ({
			id: director.id,
			slug: director.slug,
			name: director.name,
		})),
		tags: assets.tags.map((tag) => ({
			id: tag.id,
			slug: tag.slug,
			name: tag.name,
		})),
		qualities: rawScene.qualities?.sort((qualityA, qualityB) => qualityB - qualityA) || [],
		movies: assets.movies.map((movie) => ({
			id: movie.id,
			slug: movie.slug,
			title: movie.title,
			covers: movie.movie_covers?.map((cover) => curateMedia(cover)).toSorted((coverA, coverB) => coverA.index - coverB.index) || [],
		})),
		series: assets.series.map((serie) => ({
			id: serie.id,
			slug: serie.slug,
			title: serie.title,
			poster: curateMedia(serie.serie_poster),
		})),
		poster: curateMedia(assets.poster),
		trailer: curateMedia(assets.trailer),
		teaser: curateMedia(assets.teaser),
		photos: assets.photos?.map((photo) => curateMedia(photo)) || [],
		caps: assets.caps?.map((cap) => curateMedia(cap)) || [],
		stashes: assets.stashes?.map((stash) => curateStash(stash)) || [],
		createdBatchId: rawScene.created_batch_id,
		updatedBatchId: rawScene.updated_batch_id,
		isNew: assets.lastBatchId === rawScene.created_batch_id,
	};

	curatedScene.watchUrl = getAffiliateUrl(curatedScene);

	return curatedScene;
}

export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) {
	const {
		scenes,
		channels,
		actors,
		directors,
		tags,
		movies,
		series,
		posters,
		photos,
		caps,
		trailers,
		teasers,
		stashes,
		lastBatch: { id: lastBatchId },
	} = await promiseProps({
		scenes: knex('releases').whereIn('releases.id', sceneIds),
		channels: knex('releases')
			.select(
				'channels.*',
				'networks.id as network_id',
				'networks.slug as network_slug',
				'networks.name as network_name',
				'networks.type as network_type',
				knex.raw('row_to_json(affiliates) as affiliate'),
			)
			.whereIn('releases.id', sceneIds)
			.leftJoin('entities as channels', 'channels.id', 'releases.entity_id')
			.leftJoin('entities as networks', 'networks.id', 'channels.parent_id')
			.leftJoin('affiliates', knex.raw('affiliates.entity_id in (channels.id, networks.id)'))
			.groupBy('channels.id', 'networks.id', 'affiliates.id'),
		actors: knex('releases_actors')
			.select(
				'actors.*',
				'actors_meta.*',
				'countries.name as birth_country_name',
				'countries.alias as birth_country_alias',
				'releases_actors.release_id',
			)
			.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
			.leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id')
			.leftJoin('countries', 'countries.alpha2', 'actors.birth_country_alpha2')
			.whereIn('release_id', sceneIds),
		directors: knex('releases_directors')
			.whereIn('release_id', sceneIds)
			.leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'),
		tags: knex('releases_tags')
			.select('id', 'slug', 'name', 'release_id')
			.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
			.whereNotNull('tags.id')
			.whereIn('release_id', sceneIds)
			.orderBy('priority', 'desc'),
		movies: context.includePartOf ? knex('movies_scenes')
			.select('movies_scenes.scene_id', 'movies.*', knex.raw('json_agg(media) as movie_covers'))
			.leftJoin('movies', 'movies.id', 'movies_scenes.movie_id')
			.leftJoin('movies_covers', 'movies_covers.movie_id', 'movies.id')
			.leftJoin('media', 'media.id', 'movies_covers.media_id')
			.whereIn('scene_id', sceneIds)
			.groupBy('movies.id', 'movies_scenes.scene_id') : [],
		series: context.includePartOf ? knex('series_scenes')
			.select('series_scenes.scene_id', 'series.*', knex.raw('row_to_json(media) as serie_poster'))
			.leftJoin('series', 'series.id', 'series_scenes.serie_id')
			.leftJoin('series_posters', 'series_posters.serie_id', 'series.id')
			.leftJoin('media', 'media.id', 'series_posters.media_id')
			.whereIn('scene_id', sceneIds)
			.groupBy('series.id', 'series_scenes.scene_id', 'media.*') : [],
		posters: knex('releases_posters')
			.whereIn('release_id', sceneIds)
			.leftJoin('media', 'media.id', 'releases_posters.media_id'),
		photos: context.includeAssets ? knex.transaction(async (trx) => {
			if (reqUser) {
				await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
			}

			return trx('releases_photos')
				.leftJoin('media', 'media.id', 'releases_photos.media_id')
				.whereIn('release_id', sceneIds)
				.orderBy('index');
		}) : [],
		caps: context.includeAssets ? knex.transaction(async (trx) => {
			if (reqUser) {
				await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
			}

			return trx('releases_caps')
				.leftJoin('media', 'media.id', 'releases_caps.media_id')
				.whereIn('release_id', sceneIds)
				.orderBy('index');
		}) : [],
		trailers: context.includeAssets ? knex.transaction(async (trx) => {
			if (reqUser) {
				await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
			}

			return trx('releases_trailers')
				.whereIn('release_id', sceneIds)
				.leftJoin('media', 'media.id', 'releases_trailers.media_id');
		}) : [],
		teasers: context.includeAssets ? knex.transaction(async (trx) => {
			if (reqUser) {
				await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id }));
			}

			return trx('releases_teasers')
				.whereIn('release_id', sceneIds)
				.leftJoin('media', 'media.id', 'releases_teasers.media_id');
		}) : [],
		lastBatch: knex('batches')
			.select('id')
			.where('showcased', true)
			.orderBy('created_at', 'desc')
			.first(),
		stashes: reqUser
			? knexOwner('stashes_scenes')
				.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
				.where('stashes.user_id', reqUser.id)
				.whereIn('stashes_scenes.scene_id', sceneIds)
			: [],
	});

	const actorStashes = reqUser && context.actorStashes
		? await knexOwner('stashes_actors')
			.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
			.where('stashes.user_id', reqUser.id)
			.whereIn('stashes_actors.actor_id', actors.map((actor) => actor.id))
		: [];

	return sceneIds.map((sceneId) => {
		const scene = scenes.find((sceneEntry) => sceneEntry.id === sceneId);

		if (!scene) {
			return null;
		}

		const sceneChannel = channels.find((entity) => entity.id === scene.entity_id);
		const sceneActors = actors.filter((actor) => actor.release_id === sceneId);
		const sceneDirectors = directors.filter((director) => director.release_id === sceneId);
		const sceneTags = tags.filter((tag) => tag.release_id === sceneId);
		const sceneMovies = movies.filter((movie) => movie.scene_id === sceneId);
		const sceneSeries = series.filter((serie) => serie.scene_id === sceneId);
		const scenePoster = posters.find((poster) => poster.release_id === sceneId);
		const scenePhotos = photos.filter((photo) => photo.release_id === sceneId);
		const sceneCaps = caps.filter((cap) => cap.release_id === sceneId);
		const sceneTrailers = trailers.find((trailer) => trailer.release_id === sceneId);
		const sceneTeasers = teasers.find((teaser) => teaser.release_id === sceneId);
		const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId);
		const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean);

		return curateScene(scene, {
			channel: sceneChannel,
			actors: sceneActors,
			directors: sceneDirectors,
			tags: sceneTags,
			movies: sceneMovies,
			series: sceneSeries,
			poster: scenePoster,
			photos: scenePhotos,
			caps: sceneCaps,
			trailer: sceneTrailers,
			teaser: sceneTeasers,
			stashes: sceneStashes,
			actorStashes: sceneActorStashes,
			lastBatchId,
		});
	}).filter(Boolean);
}

const sqlImplied = ['scenes_stashed'];

function curateOptions(options) {
	if (options?.limit > 100) {
		throw new HttpError('Limit must be <= 100', 400);
	}

	return {
		limit: options?.limit || 30,
		page: Number(options?.page) || 1,
		aggregate: options.aggregate ?? true,
		aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true),
		aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true),
		aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true),
		index: options.index || 'scenes',
		useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false,
	};
}

async function queryManticoreSql(filters, options, _reqUser) {
	const aggSize = config.database.manticore.maxAggregateSize;

	const sqlQuery = knexManticore.raw(`
		:query:
		OPTION
			field_weights=(
				title_filtered=7,
				actors=10,
				tags=9,
				meta=6,
				channel_name=2,
				channel_slug=3,
				network_name=1,
				network_slug=1
			),
			max_matches=:maxMatches:,
			max_query_time=:maxQueryTime:
		:actorsFacet:
		:tagsFacet:
		:channelsFacet:;
		show meta;
	`, {
		query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes')
			.modify((builder) => {
				if (filters.stashId) {
					builder.select(knex.raw(`
						scenes.id as id,
						scenes.title as title,
						scenes.actor_ids as actor_ids,
						scenes.entity_ids as entity_ids,
						scenes.tag_ids as tag_ids,
						scenes.channel_id as channel_id,
						scenes.network_id as network_id,
						scenes.effective_date as effective_date,
						scenes.stashed as stashed,
						scenes.created_at,
						created_at as stashed_at,
						weight() as _score
					`));

					builder
						.innerJoin('scenes', 'scenes.id', 'scenes_stashed.scene_id')
						.where('stash_id', filters.stashId);
				} else {
					builder.select(knex.raw('*, weight() as _score'));
				}

				if (filters.query) {
					builder.whereRaw('match(\'@!title :query:\', scenes)', { query: escape(filters.query) });
				}

				filters.tagIds?.forEach((tagId) => {
					builder.where('any(tag_ids)', tagId);
				});

				if (filters.notTagIds) {
					builder.whereNotIn('tag_ids', filters.notTagIds);
				}

				filters.actorIds?.forEach((actorId) => {
					builder.where('any(actor_ids)', actorId);
				});

				if (filters.notActorIds) {
					builder.whereNotIn('actor_ids', filters.notActorIds);
				}

				if (filters.entityId) {
					builder.whereRaw('any(entity_ids) = ?', filters.entityId);

					/* manticore does not currently support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068
					builder.where((whereBuilder) => {
						whereBuilder
							.where('scenes.channel_id', filters.entityId)
							.orWhere('scenes.network_id', filters.entityId);
					});
					*/
				}

				if (filters.notEntityIds) {
					builder.whereNotIn('entity_ids', filters.notEntityIds);
				}

				if (filters.movieId) {
					builder.whereRaw('any(movie_ids) = ?', filters.movieId);
				}

				if (filters.serieId) {
					builder.whereRaw('any(serie_ids) = ?', filters.serieId);
				}

				if (typeof filters.isShowcased === 'boolean') {
					builder.where('scenes.is_showcased', filters.isShowcased);
				}

				if (filters.isShowcased) {
					builder.where('scenes.date', '>', 0);
				}

				if (!filters.scope || filters.scope === 'latest') {
					builder
						.where('effective_date', '<=', Math.round(Date.now() / 1000))
						.orderBy('scenes.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works
				} else if (filters.scope === 'upcoming') {
					builder
						.where('effective_date', '>', Math.round(Date.now() / 1000))
						.orderBy('scenes.effective_date', 'asc');
				} else if (filters.scope === 'new') {
					builder.orderBy([
						{ column: 'scenes.created_at', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'asc' },
					]);
				} else if (filters.scope === 'likes') {
					builder.orderBy([
						{ column: 'scenes.stashed', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				} else if (filters.scope === 'results') {
					builder.orderBy([
						{ column: '_score', order: 'desc' },
						{ column: 'scenes.stashed', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				} else if (filters.scope === 'stashed' && filters.stashId) {
					builder.orderBy([
						{ column: 'stashed_at', order: 'desc' },
						{ column: 'scenes.effective_date', order: 'desc' },
					]);
				} else if (filters.scope === 'oldest') {
					builder.orderBy('scenes.effective_date', 'asc');
				} else {
					builder.orderBy('scenes.effective_date', 'desc');
				}
			})
			.limit(options.limit)
			.offset((options.page - 1) * options.limit),
		// option threads=1 fixes actors, but drastically slows down performance, wait for fix
		actorsFacet: options.aggregateActors ? knex.raw('facet scenes.actor_ids order by count(*) desc limit ?', [aggSize]) : null,
		tagsFacet: options.aggregateTags ? knex.raw('facet scenes.tag_ids order by count(*) desc limit ?', [aggSize]) : null,
		channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id order by count(*) desc limit ?', [aggSize]) : null,
		maxMatches: config.database.manticore.maxMatches,
		maxQueryTime: config.database.manticore.maxQueryTime,
	}).toString();

	// manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around
	const curatedSqlQuery = filters.stashId
		? sqlQuery
		: sqlQuery.replace(/scenes\./g, '');

	if (process.env.NODE_ENV === 'development') {
		console.log(curatedSqlQuery);
	}

	const results = await utilsApi.sql(curatedSqlQuery);

	// console.log(results[0]);

	const actorIds = results
		.find((result) => (result.columns[0].actor_ids || result.columns[0]['scenes.actor_ids']) && result.columns[1]['count(*)'])
		?.data.map((row) => ({ key: row.actor_ids || row['scenes.actor_ids'], doc_count: row['count(*)'] }))
		|| [];

	const tagIds = results
		.find((result) => (result.columns[0].tag_ids || result.columns[0]['scenes.tag_ids']) && result.columns[1]['count(*)'])
		?.data.map((row) => ({ key: row.tag_ids || row['scenes.tag_ids'], doc_count: row['count(*)'] }))
		|| [];

	const channelIds = results
		.find((result) => (result.columns[0].channel_id || result.columns[0]['scenes.channel_id']) && result.columns[1]['count(*)'])
		?.data.map((row) => ({ key: row.channel_id || row['scenes.channel_id'], doc_count: row['count(*)'] }))
		|| [];

	const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0;

	return {
		scenes: results[0].data,
		total,
		aggregations: {
			actorIds,
			tagIds,
			channelIds,
		},
	};
}

function countAggregations(buckets) {
	if (!buckets) {
		return null;
	}

	return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }]));
}

export async function fetchScenes(filters, rawOptions, reqUser) {
	const options = curateOptions(rawOptions);

	console.log('filters', filters);
	console.log('options', options);

	console.time('manticore sql');
	const result = await queryManticoreSql(filters, options, reqUser);
	console.timeEnd('manticore sql');

	const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds);
	const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds);
	const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds);

	console.time('fetch aggregations');

	const [aggActors, aggTags, aggChannels] = await Promise.all([
		options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: actorCounts }) : [],
		options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(name)'), 'asc'], append: tagCounts }) : [],
		options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: channelCounts }) : [],
	]);

	console.timeEnd('fetch aggregations');

	console.time('fetch full');
	const sceneIds = result.scenes.map((scene) => Number(scene.id));
	const scenes = await fetchScenesById(sceneIds, { reqUser });
	console.timeEnd('fetch full');

	return {
		scenes,
		aggActors,
		aggTags,
		aggChannels,
		total: result.total,
		limit: options.limit,
	};
}