import config from 'config'; import { knexQuery as knex, knexOwner, knexManticore } from './knex.js'; import { utilsApi } from './manticore.js'; import { HttpError } from './errors.js'; import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js'; import { fetchTagsById } from './tags.js'; import { fetchEntitiesById } from './entities.js'; import { curateStash } from './stashes.js'; import promiseProps from '../utils/promise-props.js'; function curateMedia(media) { if (!media) { return null; } return { id: media.id, path: media.path, thumbnail: media.thumbnail, lazy: media.lazy, isS3: media.is_s3, width: media.width, height: media.height, }; } function curateMovie(rawMovie, assets) { if (!rawMovie) { return null; } return { id: rawMovie.id, title: rawMovie.title, slug: rawMovie.slug, url: rawMovie.url, date: rawMovie.date, createdAt: rawMovie.created_at, effectiveDate: rawMovie.effective_date, description: rawMovie.description, duration: rawMovie.duration, channel: { id: assets.channel.id, slug: assets.channel.slug, name: assets.channel.name, type: assets.channel.type, isIndependent: assets.channel.independent, hasLogo: assets.channel.has_logo, }, network: assets.channel.network_id ? { id: assets.channel.network_id, slug: assets.channel.network_slug, name: assets.channel.network_name, type: assets.channel.network_type, hasLogo: assets.channel.has_logo, } : null, actors: sortActorsByGender(assets.actors.map((actor) => curateActor(actor, { sceneDate: rawMovie.effective_date }))), directors: assets.directors.map((director) => ({ id: director.id, slug: director.slug, name: director.name, })), tags: assets.tags.map((tag) => ({ id: tag.id, slug: tag.slug, name: tag.name, })), poster: curateMedia(assets.poster), covers: assets.covers.map((cover) => curateMedia(cover)), photos: assets.photos.map((photo) => curateMedia(photo)), stashes: assets.stashes?.map((stash) => curateStash(stash)) || [], createdBatchId: rawMovie.created_batch_id, updatedBatchId: rawMovie.updated_batch_id, }; } export async function fetchMoviesById(movieIds, reqUser) { const { movies, channels, actors, directors, tags, covers, photos, stashes, } = await promiseProps({ movies: knex('movies') .select('movies.*', knex.raw('sum(releases.duration) as duration')) .leftJoin('movies_scenes', 'movies_scenes.movie_id', 'movies.id') .leftJoin('releases', 'releases.id', 'movies_scenes.scene_id') .whereIn('movies.id', movieIds) .groupBy('movies.id'), channels: knex('movies') .select('channels.*', 'networks.id as network_id', 'networks.slug as network_slug', 'networks.name as network_name', 'networks.type as network_type') .whereIn('movies.id', movieIds) .leftJoin('entities as channels', 'channels.id', 'movies.entity_id') .leftJoin('entities as networks', 'networks.id', 'channels.parent_id') .groupBy('channels.id', 'networks.id'), actors: knex('movies') .select( 'actors.*', 'actors_meta.*', 'releases_actors.release_id', 'movies.id as movie_id', ) .distinctOn('actors.id', 'movies.id') // cannot distinct on JSON column avatar, must specify .whereIn('movies.id', movieIds) .whereNotNull('actors.id') .leftJoin('movies_scenes', 'movies_scenes.movie_id', 'movies.id') .leftJoin('releases_actors', 'releases_actors.release_id', 'movies_scenes.scene_id') .leftJoin('actors', 'actors.id', 'releases_actors.actor_id') .leftJoin('actors_meta', 'actors_meta.actor_id', 'actors.id'), directors: knex('movies') .whereIn('movies.id', movieIds) .leftJoin('movies_scenes', 'movies_scenes.movie_id', 'movies.id') .leftJoin('releases_directors', 'releases_directors.release_id', 'movies_scenes.scene_id') .leftJoin('actors as directors', 'directors.id', 'releases_directors.director_id'), tags: knex('movies') .select('tags.id', 'tags.slug', 'tags.name', 'tags.priority', 'movies.id as movie_id') .distinct() .whereIn('movies.id', movieIds) .whereNotNull('tags.id') .leftJoin('movies_scenes', 'movies_scenes.movie_id', 'movies.id') .leftJoin('releases_tags', 'releases_tags.release_id', 'movies_scenes.scene_id') .leftJoin('tags', 'tags.id', 'releases_tags.tag_id') .orderBy('priority', 'desc'), covers: knex('movies_covers') .whereIn('movie_id', movieIds) .leftJoin('media', 'media.id', 'movies_covers.media_id') .orderBy('media.index'), photos: knex('movies') .whereIn('movies.id', movieIds) .leftJoin('movies_scenes', 'movies_scenes.movie_id', 'movies.id') .leftJoin('releases_photos', 'releases_photos.release_id', 'movies_scenes.scene_id') .leftJoin('media', 'media.id', 'releases_photos.media_id'), stashes: reqUser ? knexOwner('stashes_movies') .leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id') .where('stashes.user_id', reqUser.id) .whereIn('stashes_movies.movie_id', movieIds) : [], }); return movieIds.map((movieId) => { const movie = movies.find((movieEntry) => movieEntry.id === movieId); if (!movie) { console.warn('cannot find movie', movieId); return null; } const movieChannel = channels.find((entity) => entity.id === movie.entity_id); const movieActors = actors.filter((actor) => actor.movie_id === movieId); const movieDirectors = directors.filter((director) => director.release_id === movieId); const movieTags = tags.filter((tag) => tag.movie_id === movieId); const movieCovers = covers.filter((cover) => cover.movie_id === movieId); const moviePhotos = photos.filter((photo) => photo.release_id === movieId); const movieStashes = stashes.filter((stash) => stash.movie_id === movieId); return curateMovie(movie, { channel: movieChannel, actors: movieActors, directors: movieDirectors, tags: movieTags, covers: movieCovers, photos: moviePhotos, stashes: movieStashes, }); }).filter(Boolean); } function curateOptions(options) { if (options?.limit > 100) { throw new HttpError('Limit must be <= 100', 400); } return { limit: options?.limit || 30, page: Number(options?.page) || 1, aggregate: options.aggregate ?? true, aggregateActors: (options.aggregate ?? true) && (options.aggregateActors ?? true), aggregateTags: (options.aggregate ?? true) && (options.aggregateTags ?? true), aggregateChannels: (options.aggregate ?? true) && (options.aggregateChannels ?? true), }; } /* function buildQuery(filters = {}) { const query = { bool: { must: [], }, }; let sort = [{ effective_date: 'desc' }]; if (!filters.scope || filters.scope === 'latest') { query.bool.must.push({ range: { effective_date: { lte: Math.round(Date.now() / 1000), }, }, }); } if (filters.scope === 'upcoming') { query.bool.must.push({ range: { effective_date: { gt: Math.round(Date.now() / 1000), }, }, }); sort = [{ effective_date: 'asc' }]; } if (filters.scope === 'new') { sort = [{ created_at: 'desc' }, { effective_date: 'asc' }]; } if (filters.scope === 'likes') { sort = [{ stashed: 'desc' }, { effective_date: 'desc' }]; } if (filters.scope === 'results') { sort = [{ _score: 'desc' }, { effective_date: 'desc' }]; } if (filters.query) { query.bool.must.push({ bool: { should: [ { match: { title_filtered: filters.query } }, { match: { actors: filters.query } }, { match: { tags: filters.query } }, { match: { channel_name: filters.query } }, { match: { network_name: filters.query } }, { match: { channel_slug: filters.query } }, { match: { network_slug: filters.query } }, { match: { meta: filters.query } }, // date ], }, }); } if (filters.tagIds) { filters.tagIds.forEach((tagId) => { query.bool.must.push({ equals: { 'any(tag_ids)': tagId } }); }); } if (filters.entityId) { query.bool.must.push({ bool: { should: [ { equals: { channel_id: filters.entityId } }, { equals: { network_id: filters.entityId } }, ], }, }); } if (filters.actorIds) { filters.actorIds.forEach((actorId) => { query.bool.must.push({ equals: { 'any(actor_ids)': actorId } }); }); } if (filters.requireCover) { query.bool.must.push({ equals: { has_cover: 1, }, }); } return { query, sort }; } function buildAggregates(options) { const aggregates = {}; if (options.aggregateActors) { aggregates.actorIds = { terms: { field: 'actor_ids', size: config.database.manticore.maxAggregateSize, }, // sort: [{ 'count(*)': { order: 'desc' } }], }; } if (options.aggregateTags) { aggregates.tagIds = { terms: { field: 'tag_ids', size: config.database.manticore.maxAggregateSize, }, }; } if (options.aggregateChannels) { aggregates.channelIds = { terms: { field: 'channel_id', size: config.database.manticore.maxAggregateSize, }, }; } return aggregates; } async function queryManticoreJson(filters, options) { const { query, sort } = buildQuery(filters); console.log('query', query.bool.must); console.time('manticore'); const result = await searchApi.search({ index: 'movies', query, limit: options.limit, offset: (options.page - 1) * options.limit, sort, aggs: buildAggregates(options), options: { max_matches: config.database.manticore.maxMatches, max_query_time: config.database.manticore.maxQueryTime, field_weights: { title_filtered: 7, actors: 10, tags: 9, meta: 6, channel_name: 2, channel_slug: 3, network_name: 1, network_slug: 1, }, }, }); const movies = result.hits.hits.map((hit) => ({ id: hit._id, ...hit._source, _score: hit._score, })); return { movies, total: result.hits.total, aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])), }; } */ async function queryManticoreSql(filters, options) { const aggSize = config.database.manticore.maxAggregateSize; const sqlQuery = knexManticore.raw(` :query: OPTION field_weights=( title_filtered=7, actors=10, tags=9, meta=6, channel_name=2, channel_slug=3, network_name=1, network_slug=1 ), max_matches=:maxMatches:, max_query_time=:maxQueryTime: :actorsFacet: :tagsFacet: :channelsFacet:; show meta; `, { query: knexManticore(filters.stashId ? 'movies_stashed' : 'movies') .modify((builder) => { if (filters.stashId) { builder.select(knex.raw(` movies.id as id, movies.title as title, movies.actor_ids as actor_ids, movies.entity_ids as entity_ids, movies.tag_ids as tag_ids, movies.channel_id as channel_id, movies.network_id as network_id, movies.effective_date as effective_date, movies.stashed as stashed, movies.created_at, created_at as stashed_at, weight() as _score `)); builder .innerJoin('movies', 'movies.id', 'movies_stashed.movie_id') .where('stash_id', filters.stashId); } else { builder.select(knex.raw('*, weight() as _score')); } if (filters.query) { builder.whereRaw('match(\'@!title :query:\', movies)', { query: filters.query }); } filters.tagIds?.forEach((tagId) => { builder.where('any(tag_ids)', tagId); }); filters.actorIds?.forEach((actorId) => { builder.where('any(actor_ids)', actorId); }); if (filters.entityId) { builder.whereRaw('any(entity_ids) = ?', filters.entityId); } if (typeof filters.isShowcased === 'boolean') { builder.where('is_showcased', filters.isShowcased); } if (!filters.scope || filters.scope === 'latest') { builder .where('effective_date', '<=', Math.round(Date.now() / 1000)) .orderBy('movies.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works } else if (filters.scope === 'upcoming') { builder .where('effective_date', '>', Math.round(Date.now() / 1000)) .orderBy('movies.effective_date', 'asc'); } else if (filters.scope === 'new') { builder.orderBy([ { column: 'movies.created_at', order: 'desc' }, { column: 'movies.effective_date', order: 'asc' }, ]); } else if (filters.scope === 'likes') { builder.orderBy([ { column: 'movies.stashed', order: 'desc' }, { column: 'movies.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'results') { builder.orderBy([ { column: '_score', order: 'desc' }, { column: 'movies.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'stashed' && filters.stashId) { builder.orderBy([ { column: 'stashed_at', order: 'desc' }, { column: 'movies.effective_date', order: 'desc' }, ]); } else { builder.orderBy('movies.effective_date', 'desc'); } }) .limit(options.limit) .offset((options.page - 1) * options.limit) .toString(), // option threads=1 fixes actors, but drastically slows down performance, wait for fix actorsFacet: options.aggregateActors ? knex.raw('facet movies.actor_ids order by count(*) desc limit ?', [aggSize]) : null, tagsFacet: options.aggregateTags ? knex.raw('facet movies.tag_ids order by count(*) desc limit ?', [aggSize]) : null, channelsFacet: options.aggregateChannels ? knex.raw('facet movies.channel_id order by count(*) desc limit ?', [aggSize]) : null, maxMatches: config.database.manticore.maxMatches, maxQueryTime: config.database.manticore.maxQueryTime, }).toString(); // manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around const curatedSqlQuery = filters.stashId ? sqlQuery : sqlQuery.replace(/movies\./g, ''); if (process.env.NODE_ENV === 'development') { console.log(curatedSqlQuery); } const results = await utilsApi.sql(curatedSqlQuery); // console.log(results[0]); const actorIds = results .find((result) => (result.columns[0].actor_ids || result.columns[0]['movies.actor_ids']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.actor_ids || row['movies.actor_ids'], doc_count: row['count(*)'] })) || []; const tagIds = results .find((result) => (result.columns[0].tag_ids || result.columns[0]['movies.tag_ids']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.tag_ids || row['movies.tag_ids'], doc_count: row['count(*)'] })) || []; const channelIds = results .find((result) => (result.columns[0].channel_id || result.columns[0]['movies.channel_id']) && result.columns[1]['count(*)']) ?.data.map((row) => ({ key: row.channel_id || row['movies.channel_id'], doc_count: row['count(*)'] })) || []; const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0; return { movies: results[0].data, total, aggregations: { actorIds, tagIds, channelIds, }, }; } function countAggregations(buckets) { if (!buckets) { return null; } return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }])); } export async function fetchMovies(filters, rawOptions, reqUser) { const options = curateOptions(rawOptions); console.log(options); console.log(filters); const result = await queryManticoreSql(filters, options); const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds); const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds); const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds); const [aggActors, aggTags, aggChannels] = await Promise.all([ options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: actorCounts }) : [], options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: [knex.raw('lower(name)'), 'asc'], append: tagCounts }) : [], options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['slug', 'asc'], append: channelCounts }) : [], ]); const movieIds = result.movies.map((movie) => Number(movie.id)); const movies = await fetchMoviesById(movieIds, reqUser); return { movies, aggActors, aggTags, aggChannels, total: result.total, limit: options.limit, }; }