From 72af9add7d079ed41b19a4f45b3412901759bd71 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 24 Mar 2024 23:36:25 +0100 Subject: [PATCH] Added movies stash page. --- components/movies/movies.vue | 488 ++++++++++++++++++ components/scenes/scenes.vue | 2 +- pages/movies/+Page.vue | 474 +---------------- pages/movies/+onBeforeRender.js | 1 - .../@username/@stashSlug/movies/+Page.vue | 4 +- .../@stashSlug/movies/+onBeforeRender.js | 26 + src/actors.js | 2 +- src/movies.js | 229 ++++++-- src/scenes.js | 11 +- src/tools/sync-stashes.js | 33 +- src/web/movies.js | 1 + 11 files changed, 739 insertions(+), 532 deletions(-) create mode 100644 components/movies/movies.vue diff --git a/components/movies/movies.vue b/components/movies/movies.vue new file mode 100644 index 0000000..0031aa6 --- /dev/null +++ b/components/movies/movies.vue @@ -0,0 +1,488 @@ + + + + + diff --git a/components/scenes/scenes.vue b/components/scenes/scenes.vue index 57e798f..a655d84 100644 --- a/components/scenes/scenes.vue +++ b/components/scenes/scenes.vue @@ -100,7 +100,7 @@ >
    • diff --git a/pages/movies/+Page.vue b/pages/movies/+Page.vue index cdb11e9..eda5d30 100644 --- a/pages/movies/+Page.vue +++ b/pages/movies/+Page.vue @@ -1,477 +1,9 @@ - - diff --git a/pages/movies/+onBeforeRender.js b/pages/movies/+onBeforeRender.js index ff33ed6..987575f 100644 --- a/pages/movies/+onBeforeRender.js +++ b/pages/movies/+onBeforeRender.js @@ -8,7 +8,6 @@ export async function onBeforeRender(pageContext) { }), { page: Number(pageContext.routeParams.page) || 1, limit: Number(pageContext.urlParsed.search.limit) || 50, - aggregate: true, }); const { diff --git a/pages/stashes/@username/@stashSlug/movies/+Page.vue b/pages/stashes/@username/@stashSlug/movies/+Page.vue index f318230..ca738dd 100644 --- a/pages/stashes/@username/@stashSlug/movies/+Page.vue +++ b/pages/stashes/@username/@stashSlug/movies/+Page.vue @@ -1,10 +1,10 @@ diff --git a/pages/stashes/@username/@stashSlug/movies/+onBeforeRender.js b/pages/stashes/@username/@stashSlug/movies/+onBeforeRender.js index 159a43e..5195050 100644 --- a/pages/stashes/@username/@stashSlug/movies/+onBeforeRender.js +++ b/pages/stashes/@username/@stashSlug/movies/+onBeforeRender.js @@ -1,17 +1,43 @@ import { render } from 'vike/abort'; /* eslint-disable-line import/extensions */ import { fetchStashByUsernameAndSlug } from '#/src/stashes.js'; +import { fetchMovies } from '#/src/movies.js'; +import { curateMoviesQuery } from '#/src/web/movies.js'; import { HttpError } from '#/src/errors.js'; export async function onBeforeRender(pageContext) { try { const stash = await fetchStashByUsernameAndSlug(pageContext.routeParams.username, pageContext.routeParams.stashSlug, pageContext.user); + const movieResults = await fetchMovies(await curateMoviesQuery({ + ...pageContext.urlQuery, + scope: pageContext.routeParams.scope || 'latest', + stashId: stash.id, + }), { + page: Number(pageContext.routeParams.page) || 1, + limit: Number(pageContext.urlParsed.search.limit) || 50, + }); + + const { + movies, + aggActors, + aggTags, + aggChannels, + total, + limit, + } = movieResults; + return { pageContext: { title: `${stash.name} by ${stash.user.username}`, pageProps: { stash, + movies, + aggActors, + aggTags, + aggChannels, + total, + limit, }, }, }; diff --git a/src/actors.js b/src/actors.js index 3b7e7cd..24d969d 100644 --- a/src/actors.js +++ b/src/actors.js @@ -442,7 +442,7 @@ async function queryManticoreSql(filters, options, _reqUser) { ]); } else if (options.order?.[0] === 'results') { builder.orderBy([ - { column: 'actors._score', order: options.order[1] }, + { column: '_score', order: options.order[1] }, { column: 'actors.slug', order: 'asc' }, ]); } else if (options.order?.[0] === 'stashed' && filters.stashId) { diff --git a/src/movies.js b/src/movies.js index a8e3661..6287e47 100644 --- a/src/movies.js +++ b/src/movies.js @@ -1,7 +1,7 @@ import config from 'config'; -import knex from './knex.js'; -import { searchApi } from './manticore.js'; +import { knexQuery as knex, knexManticore } from './knex.js'; +import { utilsApi } from './manticore.js'; import { HttpError } from './errors.js'; import { fetchActorsById, curateActor, sortActorsByGender } from './actors.js'; import { fetchTagsById } from './tags.js'; @@ -166,6 +166,7 @@ function curateOptions(options) { }; } +/* function buildQuery(filters = {}) { const query = { bool: { @@ -257,16 +258,6 @@ function buildQuery(filters = {}) { }); } - /* tag filter - must_not: [ - { - in: { - 'any(tag_ids)': [101, 180, 32], - }, - }, - ], - */ - return { query, sort }; } @@ -304,20 +295,9 @@ function buildAggregates(options) { return aggregates; } -function countAggregations(buckets) { - if (!buckets) { - return null; - } - - return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }])); -} - -export async function fetchMovies(filters, rawOptions) { - const options = curateOptions(rawOptions); +async function queryManticoreJson(filters, options) { const { query, sort } = buildQuery(filters); - console.log('filters', filters); - console.log('options', options); console.log('query', query.bool.must); console.time('manticore'); @@ -345,23 +325,202 @@ export async function fetchMovies(filters, rawOptions) { }, }); - console.timeEnd('manticore'); + const movies = result.hits.hits.map((hit) => ({ + id: hit._id, + ...hit._source, + _score: hit._score, + })); - const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds?.buckets); - const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds?.buckets); - const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds?.buckets); + return { + movies, + total: result.hits.total, + aggregations: result.aggregations && Object.fromEntries(Object.entries(result.aggregations).map(([key, { buckets }]) => [key, buckets])), + }; +} +*/ - console.time('fetch aggregations'); +async function queryManticoreSql(filters, options) { + const aggSize = config.database.manticore.maxAggregateSize; + + const sqlQuery = knexManticore.raw(` + :query: + OPTION + field_weights=( + title_filtered=7, + actors=10, + tags=9, + meta=6, + channel_name=2, + channel_slug=3, + network_name=1, + network_slug=1 + ), + max_matches=:maxMatches:, + max_query_time=:maxQueryTime: + :actorsFacet: + :tagsFacet: + :channelsFacet:; + show meta; + `, { + query: knexManticore(filters.stashId ? 'movies_stashed' : 'movies') + .modify((builder) => { + if (filters.stashId) { + builder.select(knex.raw(` + movies.id as id, + movies.title as title, + movies.actor_ids as actor_ids, + movies.entity_ids as entity_ids, + movies.tag_ids as tag_ids, + movies.channel_id as channel_id, + movies.network_id as network_id, + movies.effective_date as effective_date, + movies.stashed as stashed, + movies.created_at, + created_at as stashed_at, + weight() as _score + `)); + + builder + .innerJoin('movies', 'movies.id', 'movies_stashed.movie_id') + .where('stash_id', filters.stashId); + } else { + builder.select(knex.raw('*, weight() as _score')); + } + + if (filters.query) { + builder.whereRaw('match(\'@!title :query:\', movies)', { query: filters.query }); + } + + filters.tagIds?.forEach((tagId) => { + builder.where('any(tag_ids)', tagId); + }); + + filters.actorIds?.forEach((actorId) => { + builder.where('any(actor_ids)', actorId); + }); + + if (filters.entityId) { + builder.whereRaw('any(entity_ids) = ?', filters.entityId); + } + + if (typeof filters.isShowcased === 'boolean') { + builder.where('is_showcased', filters.isShowcased); + } + + if (!filters.scope || filters.scope === 'latest') { + builder + .where('effective_date', '<=', Math.round(Date.now() / 1000)) + .orderBy('movies.effective_date', 'desc'); // can't seem to use alias if it matches column-name? behavior not fully understand, but this works + } else if (filters.scope === 'upcoming') { + builder + .where('effective_date', '>', Math.round(Date.now() / 1000)) + .orderBy('movies.effective_date', 'asc'); + } else if (filters.scope === 'new') { + builder.orderBy([ + { column: 'movies.created_at', order: 'desc' }, + { column: 'movies.effective_date', order: 'asc' }, + ]); + } else if (filters.scope === 'likes') { + builder.orderBy([ + { column: 'movies.stashed', order: 'desc' }, + { column: 'movies.effective_date', order: 'desc' }, + ]); + } else if (filters.scope === 'results') { + builder.orderBy([ + { column: '_score', order: 'desc' }, + { column: 'movies.effective_date', order: 'desc' }, + ]); + } else if (filters.scope === 'stashed' && filters.stashId) { + builder.orderBy([ + { column: 'stashed_at', order: 'desc' }, + { column: 'movies.effective_date', order: 'desc' }, + ]); + } else { + builder.orderBy('movies.effective_date', 'desc'); + } + }) + .limit(options.limit) + .offset((options.page - 1) * options.limit) + .toString(), + // option threads=1 fixes actors, but drastically slows down performance, wait for fix + actorsFacet: options.aggregateActors ? knex.raw('facet movies.actor_ids order by count(*) desc limit ?', [aggSize]) : null, + tagsFacet: options.aggregateTags ? knex.raw('facet movies.tag_ids order by count(*) desc limit ?', [aggSize]) : null, + channelsFacet: options.aggregateChannels ? knex.raw('facet movies.channel_id order by count(*) desc limit ?', [aggSize]) : null, + maxMatches: config.database.manticore.maxMatches, + maxQueryTime: config.database.manticore.maxQueryTime, + }).toString(); + + // manticore does not seem to accept table.column syntax if 'table' is primary (yet?), crude work-around + const curatedSqlQuery = filters.stashId + ? sqlQuery + : sqlQuery.replace(/movies\./g, ''); + + if (process.env.NODE_ENV === 'development') { + console.log(curatedSqlQuery); + } + + const results = await utilsApi.sql(curatedSqlQuery); + + // console.log(results[0]); + + const actorIds = results + .find((result) => (result.columns[0].actor_ids || result.columns[0]['movies.actor_ids']) && result.columns[1]['count(*)']) + ?.data.map((row) => ({ key: row.actor_ids || row['movies.actor_ids'], doc_count: row['count(*)'] })) + || []; + + const tagIds = results + .find((result) => (result.columns[0].tag_ids || result.columns[0]['movies.tag_ids']) && result.columns[1]['count(*)']) + ?.data.map((row) => ({ key: row.tag_ids || row['movies.tag_ids'], doc_count: row['count(*)'] })) + || []; + + const channelIds = results + .find((result) => (result.columns[0].channel_id || result.columns[0]['movies.channel_id']) && result.columns[1]['count(*)']) + ?.data.map((row) => ({ key: row.channel_id || row['movies.channel_id'], doc_count: row['count(*)'] })) + || []; + + const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0; + + return { + movies: results[0].data, + total, + aggregations: { + actorIds, + tagIds, + channelIds, + }, + }; +} + +function countAggregations(buckets) { + if (!buckets) { + return null; + } + + return Object.fromEntries(buckets.map((bucket) => [bucket.key, { count: bucket.doc_count }])); +} + +export async function fetchMovies(filters, rawOptions) { + const options = curateOptions(rawOptions); + + console.log(options); + console.log(filters); + + const result = await queryManticoreSql(filters, options); + + const actorCounts = options.aggregateActors && countAggregations(result.aggregations?.actorIds); + const tagCounts = options.aggregateTags && countAggregations(result.aggregations?.tagIds); + const channelCounts = options.aggregateChannels && countAggregations(result.aggregations?.channelIds); const [aggActors, aggTags, aggChannels] = await Promise.all([ - options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [], - options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [], - options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.buckets.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [], + options.aggregateActors ? fetchActorsById(result.aggregations.actorIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: actorCounts }) : [], + options.aggregateTags ? fetchTagsById(result.aggregations.tagIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: tagCounts }) : [], + options.aggregateChannels ? fetchEntitiesById(result.aggregations.channelIds.map((bucket) => bucket.key), { order: ['name', 'asc'], append: channelCounts }) : [], ]); - console.timeEnd('fetch aggregations'); + console.log(result.aggregations); + console.log(aggActors); - const movieIds = result.hits.hits.map((hit) => Number(hit._id)); + const movieIds = result.movies.map((movie) => Number(movie.id)); const movies = await fetchMoviesById(movieIds); return { @@ -369,7 +528,7 @@ export async function fetchMovies(filters, rawOptions) { aggActors, aggTags, aggChannels, - total: result.hits.total, + total: result.total, limit: options.limit, }; } diff --git a/src/scenes.js b/src/scenes.js index ee4cd05..a8698e9 100644 --- a/src/scenes.js +++ b/src/scenes.js @@ -190,8 +190,6 @@ export async function fetchScenesById(sceneIds, { reqUser, ...context } = {}) { const sceneStashes = stashes.filter((stash) => stash.scene_id === sceneId); const sceneActorStashes = sceneActors.map((actor) => actorStashes.find((stash) => stash.actor_id === actor.id)).filter(Boolean); - console.log(sceneActors); - return curateScene(scene, { channel: sceneChannel, actors: sceneActors, @@ -411,6 +409,7 @@ async function queryManticoreSql(filters, options, _reqUser) { scenes.id as id, scenes.title as title, scenes.actor_ids as actor_ids, + scenes.entity_ids as entity_ids, scenes.tag_ids as tag_ids, scenes.channel_id as channel_id, scenes.network_id as network_id, @@ -441,11 +440,15 @@ async function queryManticoreSql(filters, options, _reqUser) { }); if (filters.entityId) { + builder.whereRaw('any(entity_ids) = ?', filters.entityId); + + /* manticore does not support OR if both left and right table are queried https://github.com/manticoresoftware/manticoresearch/issues/1978#issuecomment-2010470068 builder.where((whereBuilder) => { whereBuilder .where('scenes.channel_id', filters.entityId) .orWhere('scenes.network_id', filters.entityId); }); + */ } if (typeof filters.isShowcased === 'boolean') { @@ -472,7 +475,7 @@ async function queryManticoreSql(filters, options, _reqUser) { ]); } else if (filters.scope === 'results') { builder.orderBy([ - { column: 'scenes._score', order: 'desc' }, + { column: '_score', order: 'desc' }, { column: 'scenes.effective_date', order: 'desc' }, ]); } else if (filters.scope === 'stashed' && filters.stashId) { @@ -523,7 +526,7 @@ async function queryManticoreSql(filters, options, _reqUser) { ?.data.map((row) => ({ key: row.channel_id || row['scenes.channel_id'], doc_count: row['count(*)'] })) || []; - const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found').Value); + const total = Number(results.at(-1).data.find((entry) => entry.Variable_name === 'total_found')?.Value) || 0; return { scenes: results[0].data, diff --git a/src/tools/sync-stashes.js b/src/tools/sync-stashes.js index ee38dc6..a278763 100644 --- a/src/tools/sync-stashes.js +++ b/src/tools/sync-stashes.js @@ -1,31 +1,29 @@ -import { indexApi } from '../manticore.js'; +import { indexApi, utilsApi } from '../manticore.js'; import { knexOwner as knex } from '../knex.js'; import chunk from '../utils/chunk.js'; -async function syncActorStashes() { - const stashes = await knex('stashes_actors') +async function syncStashes(domain = 'scene') { + await utilsApi.sql(`truncate table ${domain}s_stashed`); + + const stashes = await knex(`stashes_${domain}s`) .select( - 'stashes_actors.id as stashed_id', - 'stashes_actors.actor_id', + `stashes_${domain}s.id as stashed_id`, + `stashes_${domain}s.${domain}_id`, 'stashes.id as stash_id', 'stashes.user_id as user_id', - 'stashes_actors.created_at as created_at', + `stashes_${domain}s.created_at as created_at`, ) - .leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id'); - - if (stashes.length > 0) { - console.log(stashes); - } + .leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`); await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => { await chain; const stashDocs = stashChunk.map((stash) => ({ replace: { - index: 'actors_stashed', + index: `${domain}s_stashed`, id: stash.stashed_id, doc: { - actor_id: stash.actor_id, + [`${domain}_id`]: stash[`${domain}_id`], stash_id: stash.stash_id, user_id: stash.user_id, created_at: Math.round(stash.created_at.getTime() / 1000), @@ -33,16 +31,17 @@ async function syncActorStashes() { }, })); - console.log(stashDocs); - await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n')); - console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} actor stashes`); + console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`); }, Promise.resolve()); } async function init() { - await syncActorStashes(); + await syncStashes('scene'); + await syncStashes('actor'); + await syncStashes('movie'); + console.log('Done!'); knex.destroy(); diff --git a/src/web/movies.js b/src/web/movies.js index 2bc5c89..7b4228d 100644 --- a/src/web/movies.js +++ b/src/web/movies.js @@ -12,6 +12,7 @@ export async function curateMoviesQuery(query) { tagIds: await getIdsBySlug([query.tagSlug, ...(query.tags?.split(',') || [])], 'tags'), entityId: query.e ? await getIdsBySlug([query.e], 'entities').then(([id]) => id) : query.entityId, requireCover: query.cover, + stashId: Number(query.stashId) || null, }; }