import config from 'config'; import util from 'util'; /* eslint-disable-line no-unused-vars */ import { knexQuery as knex, knexOwner, knexManticore } from './knex.js'; import { utilsApi } from './manticore.js'; import { HttpError } from './errors.js'; import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js'; import { fetchTagsById } from './tags.js'; import { fetchEntitiesById } from './entities.js'; import { curateStash } from './stashes.js'; import { curateMedia } from './media.js'; import escape from '../utils/escape-manticore.js'; import promiseProps from '../utils/promise-props.js'; function curateScene(rawScene, assets) { if (!rawScene) { return null; } return { id: rawScene.id, title: rawScene.title, slug: rawScene.slug, url: rawScene.url, date: rawScene.date, createdAt: rawScene.created_at, effectiveDate: rawScene.effective_date, description: rawScene.description, duration: rawScene.duration, shootId: rawScene.shoot_id, channel: { id: assets.channel.id, slug: assets.channel.slug, name: assets.channel.name, type: assets.channel.type, isIndependent: assets.channel.independent, hasLogo: assets.channel.has_logo, }, network: assets.channel.network_id ? { id: assets.channel.network_id, slug: assets.channel.network_slug, name: assets.channel.network_name, type: assets.channel.network_type, hasLogo: assets.channel.has_logo, } : null, actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, { sceneDate: rawScene.effective_date, stashes: assets.actorStashes.filter((actorStash) => actorStash.actor_id === actor.id), })), { title: rawScene.title }), directors: assets.directors.map((director) => ({ id: director.id, slug: director.slug, name: director.name, })), tags: assets.tags.map((tag) => ({ id: tag.id, slug: tag.slug, name: tag.name, })), qualities: rawScene.qualities?.sort((qualityA, qualityB) => qualityB - qualityA) || [], movies: assets.movies.map((movie) => ({ id: movie.id, slug: movie.slug, title: movie.title, covers: movie.movie_covers?.map((cover) => curateMedia(cover)).toSorted((coverA, coverB) => coverA.index - coverB.index) || [], })), series: assets.series.map((serie) => ({ id: serie.id, slug: serie.slug, title: serie.title, poster: curateMedia(serie.serie_poster), })), poster: curateMedia(assets.poster), trailer: curateMedia(assets.trailer), teaser: curateMedia(assets.teaser), photos: assets.photos?.map((photo) => curateMedia(photo)) || [], caps: assets.caps?.map((cap) => curateMedia(cap)) || [], stashes: assets.stashes?.map((stash) => curateStash(stash)) || [], createdBatchId: rawScene.created_batch_id, updatedBatchId: rawScene.updated_batch_id, isNew: assets.lastBatchId === rawScene.created_batch_id, }; } export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) { const { scenes, channels, actors, directors, tags, movies, series, posters, photos, caps, trailers, teasers, stashes, lastBatch: { id: lastBatchId }, } = await promiseProps({ scenes: knex('releases').whereIn('releases.id', sceneIds), channels: knex('releases') .select('channels.*', 'networks.id as network_id', 'networks.slug as network_slug', 'networks.name as network_name', 'networks.type as network_type') .whereIn('releases.id', sceneIds) .leftJoin('entities as channels', 'channels.id', 'releases.entity_id') .leftJoin('entities as networks', 'networks.id', 'channels.parent_id') .groupBy('channels.id', 'networks.id'), actors: knex('releases_actors') .select( 'actors.*', 'actors_meta.*', 'countries.name as birth_country_name', 'countries.alias as birth_country_alias', 'releases_actors.release_id', ) .leftJoin('actors', 'actors.id', 'releases_actors.actor_id') .leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id') .leftJoin('countries', 'countries.alpha2', 'actors.birth_country_alpha2') .whereIn('release_id', sceneIds), directors: knex('releases_directors') .whereIn('release_id', sceneIds) .leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'), tags: knex('releases_tags') .select('id', 'slug', 'name', 'release_id') .leftJoin('tags', 'tags.id', 'releases_tags.tag_id') .whereNotNull('tags.id') .whereIn('release_id', sceneIds) .orderBy('priority', 'desc'), movies: context.includePartOf ? knex('movies_scenes') .select('movies_scenes.scene_id', 'movies.*', knex.raw('json_agg(media) as movie_covers')) .leftJoin('movies', 'movies.id', 'movies_scenes.movie_id') .leftJoin('movies_covers', 'movies_covers.movie_id', 'movies.id') .leftJoin('media', 'media.id', 'movies_covers.media_id') .whereIn('scene_id', sceneIds) .groupBy('movies.id', 'movies_scenes.scene_id') : [], series: context.includePartOf ? knex('series_scenes') .select('series_scenes.scene_id', 'series.*', knex.raw('row_to_json(media) as serie_poster')) .leftJoin('series', 'series.id', 'series_scenes.serie_id') .leftJoin('series_posters', 'series_posters.serie_id', 'series.id') .leftJoin('media', 'media.id', 'series_posters.media_id') .whereIn('scene_id', sceneIds) .groupBy('series.id', 'series_scenes.scene_id', 'media.*') : [], posters: knex('releases_posters') .whereIn('release_id', sceneIds) .leftJoin('media', 'media.id', 'releases_posters.media_id'), photos: context.includeAssets ? knex.transaction(async (trx) => { if (reqUser) { await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id })); } return trx('releases_photos') .leftJoin('media', 'media.id', 'releases_photos.media_id') .whereIn('release_id', sceneIds) .orderBy('index'); }) : [], caps: context.includeAssets ? knex.transaction(async (trx) => { if (reqUser) { await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id })); } return trx('releases_caps') .leftJoin('media', 'media.id', 'releases_caps.media_id') .whereIn('release_id', sceneIds) .orderBy('index'); }) : [], trailers: context.includeAssets ? knex.transaction(async (trx) => { if (reqUser) { await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id })); } return trx('releases_trailers') .whereIn('release_id', sceneIds) .leftJoin('media', 'media.id', 'releases_trailers.media_id'); }) : [], teasers: context.includeAssets ? knex.transaction(async (trx) => { if (reqUser) { await trx.select(knex.raw('set_config(\'user.id\', :userId, true)', { userId: reqUser.id })); } return trx('releases_teasers') .whereIn('release_id', sceneIds) .leftJoin('media', 'media.id', 'releases_teasers.media_id'); }) : [], lastBatch: knex('batches') .select('id') .where('showcased', true) .orderBy('created_at', 'desc') .first(), stashes: reqUser ? knexOwner('stashes_scenes') .leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id') .where('stashes.user_id', reqUser.id) .whereIn('stashes_scenes.scene_id', sceneIds) : [], }); const actorStashes = reqUser && context.actorStashes ? await knexOwner('stashes_actors') .leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id') .where('stashes.user_id', reqUser.id) .whereIn('stashes_actors.actor_id', actors.map((actor) => actor.id)) : []; return sceneIds.map((sceneId) => { const scene = scenes.find((sceneEntry) => sceneEntry.id === sceneId); if (!scene) { return null; } const sceneChannel = channels.find((entity) => entity.id === scene.entity_id); const sceneActors = actors.filter((actor) => actor.release_id === sceneId); const sceneDirectors = directors.filter((director) => director.release_id === sceneId); const sceneTags = tags.filter((tag) => tag.release_id === sceneId); const sceneMovies = movies.filter((movie) => movie.scene_id === sceneId); const sceneSeries = series.filter((serie) => serie.scene_id === sceneId); const scenePoster = posters.find((poster) => poster.release_id === sceneId); const scenePhotos = photos.filter((photo) => photo.release_id === sceneId); const sceneCaps = caps.filter((cap) => cap.release_id === sceneId); const sceneTrailers = trailers.find((trailer) => trailer.release_id === sceneId); const sceneTeasers = teasers.find((teaser) => teaser.release_id === sceneId); const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId); const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean); return curateScene(scene, { channel: sceneChannel, actors: sceneActors, directors: sceneDirectors, tags: sceneTags, movies: sceneMovies, series: sceneSeries, poster: scenePoster, photos: scenePhotos, caps: sceneCaps, trailer: sceneTrailers, teaser: sceneTeasers, stashes: sceneStashes, actorStashes: sceneActorStashes, lastBatchId, }); }).filter(Boolean); } const sqlImplied = ['scenes_stashed']; function curateOptions(options) { if (options?.limit > 100) { throw new HttpError('Limit must be <= 100', 400); } return { limit: options?.limit || 30, page: Number(options?.page) || 1, aggregate: options.aggregate ?? true, aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true), aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true), aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true), index: options.index || 'scenes', useSql: options.useSql || (typeof options.useSql === 'undefined' && sqlImplied.includes(options.index)) || false, }; } async function queryManticoreSql(filters, options, _reqUser) { const aggSize = config.database.manticore.maxAggregateSize; const sqlQuery = knexManticore.raw(` :query: OPTION field_weights=( title_filtered=7, actors=10, tags=9, meta=6, channel_name=2, channel_slug=3, network_name=1, network_slug=1 ), max_matches=:maxMatches:, max_query_time=:maxQueryTime: :actorsFacet: :tagsFacet: :channelsFacet:; show meta; `, { query: knexManticore(filters.stashId ? 'scenes_stashed' : 'scenes') .modify((builder) => { if (filters.stashId) { builder.select(knex.raw(` scenes.id as id, scenes.title as title, scenes.actor_ids as actor_ids, scenes.entity_ids as entity_ids, scenes.tag_ids as tag_ids, scenes.channel_id as channel_id, scenes.network_id as network_id, scenes.effective_date as effective_date, scenes.stashed as stashed, scenes.created_at, created_at as stashed_at, weight() as _score `)); builder .innerJoin('scenes', 'scenes.id', 'scenes_stashed.scene_id') .where('stash_id', filters.stashId); } else { builder.select(knex.raw('*, weight() as _score')); } if (filters.query) { builder.whereRaw('match(\'@!title :query:\', scenes)', { query: escape(filters.query) }); } filters.tagIds?.forEach((tagId) => { builder.where('any(tag_ids)', tagId); }); if (filters.notTagIds) { builder.whereNotIn('tag_ids', filters.notTagIds); } filters.actorIds?.forEach((actorId) => { builder.where('any(actor_ids)', actorId); }); if (filters.notActorIds) { builder.whereNotIn('actor_ids', filters.notActorIds); } if (filters.entityId) { builder.whereRaw('any(entity_ids) = ?', filters.entityId); /* manticore does not currently support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068 builder.where((whereBuilder) => { whereBuilder .where('scenes.channel_id', filters.entityId) .orWhere('scenes.network_id', filters.entityId); }); */ } if (filters.notEntityIds) { builder.whereNotIn('entity_ids', filters.notEntityIds); } if (filters.movieId) { builder.whereRaw('any(movie_ids) = ?', filters.movieId); } if (filters.serieId) { builder.whereRaw('any(serie_ids) = ?', filters.serieId); } if (typeof filters.isShowcased === 'boolean') { builder.where('scenes.is_showcased', filters.isShowcased); } if (filters.isShowcased) { builder.where('scenes.date', '>', 0); } if (!filters.scope || filters.scope === 'latest') { builder .where('effective_date', '<=', Math.round(Date.now() / 1000)) .orderBy('scenes.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works } else if (filters.scope === 'upcoming') { builder .where('effective_date', '>', Math.round(Date.now() / 1000)) .orderBy('scenes.effective_date', 'asc'); } else if (filters.scope === 'new') { builder.orderBy([ { column: 'scenes.created_at', order: 'desc' }, { column: 'scenes.effective_date', order: 'asc' }, ]); } else if (filters.scope === 'likes') { builder.orderBy([ { column: 'scenes.stashed', order: 'desc' }, { column: 'scenes.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'results') { builder.orderBy([ { column: '_score', order: 'desc' }, { column: 'scenes.stashed', order: 'desc' }, { column: 'scenes.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'stashed' && filters.stashId) { builder.orderBy([ { column: 'stashed_at', order: 'desc' }, { column: 'scenes.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'oldest') { builder.orderBy('scenes.effective_date', 'asc'); } else { builder.orderBy('scenes.effective_date', 'desc'); } }) .limit(options.limit) .offset((options.page - 1) * options.limit), // option threads=1 fixes actors, but drastically slows down performance, wait for fix actorsFacet: options.aggregateActors ? knex.raw('facet scenes.actor_ids order by count(*) desc limit ?', [aggSize]) : null, tagsFacet: options.aggregateTags ? knex.raw('facet scenes.tag_ids order by count(*) desc limit ?', [aggSize]) : null, channelsFacet: options.aggregateChannels ? knex.raw('facet scenes.channel_id order by count(*) desc limit ?', [aggSize]) : null, maxMatches: config.database.manticore.maxMatches, maxQueryTime: config.database.manticore.maxQueryTime, }).toString(); // manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around const curatedSqlQuery = filters.stashId ? sqlQuery : sqlQuery.replace(/scenes\./g, ''); if (process.env.NODE_ENV === 'development') { console.log(curatedSqlQuery); } const results = await utilsApi.sql(curatedSqlQuery); // console.log(results[0]); const actorIds = results .find((result) => (result.columns[0].actor_ids || result.columns[0]['scenes.actor_ids']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.actor_ids || row['scenes.actor_ids'], doc_count: row['count(*)'] })) || []; const tagIds = results .find((result) => (result.columns[0].tag_ids || result.columns[0]['scenes.tag_ids']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.tag_ids || row['scenes.tag_ids'], doc_count: row['count(*)'] })) || []; const channelIds = results .find((result) => (result.columns[0].channel_id || result.columns[0]['scenes.channel_id']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.channel_id || row['scenes.channel_id'], doc_count: row['count(*)'] })) || []; const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0; return { scenes: results[0].data, total, aggregations: { actorIds, tagIds, channelIds, }, }; } function countAggregations(buckets) { if (!buckets) { return null; } return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }])); } export async function fetchScenes(filters, rawOptions, reqUser) { const options = curateOptions(rawOptions); console.log('filters', filters); console.log('options', options); console.time('manticore sql'); const result = await queryManticoreSql(filters, options, reqUser); console.timeEnd('manticore sql'); const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds); const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds); const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds); console.time('fetch aggregations'); const [aggActors, aggTags, aggChannels] = await Promise.all([ options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: actorCounts }) : [], options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(name)'), 'asc'], append: tagCounts }) : [], options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: channelCounts }) : [], ]); console.timeEnd('fetch aggregations'); console.time('fetch full'); const sceneIds = result.scenes.map((scene) => Number(scene.id)); const scenes = await fetchScenesById(sceneIds, { reqUser }); console.timeEnd('fetch full'); return { scenes, aggActors, aggTags, aggChannels, total: result.total, limit: options.limit, }; }