diff --git a/config/default.js b/config/default.js index 639ab1a9..4cdef114 100755 --- a/config/default.js +++ b/config/default.js @@ -199,6 +199,12 @@ module.exports = { // source: 'http://nsfw.unknown.name/random', }, }, + webApi: { + enabled: true, + address: 'http://localhost:5100/api', + apiUserId: 1, + apiKey: null, + }, proxy: { enable: false, test: 'https://api.ipify.org?format=json', diff --git a/migrations/20260608053154_sync_abilities.js b/migrations/20260608053154_sync_abilities.js new file mode 100644 index 00000000..cca29087 --- /dev/null +++ b/migrations/20260608053154_sync_abilities.js @@ -0,0 +1,87 @@ +exports.up = async function(knex) { + await knex.schema.createTable('queue', (table) => { + table.increments('id'); + + table.string('domain'); + table.specificType('item_ids', 'integer array'); + + table.text('comment'); + + table.datetime('created_at') + .defaultTo(knex.fn.now()); + }); + + await knex('users_roles') + .update('abilities', JSON.stringify([ + { subject: 'scene', action: 'create' }, + { subject: 'scene', action: 'update' }, + { subject: 'scene', action: 'delete' }, + { subject: 'actor', action: 'create' }, + { subject: 'actor', action: 'update' }, + { subject: 'actor', action: 'delete' }, + { subject: 'actor', action: 'merge' }, + { subject: 'sync' }, + { subject: 'plainUrls' }, + ])) + .where('role', 'admin'); + + await knex.raw(` + DROP TABLE IF EXISTS releases_search CASCADE; + DROP TABLE IF EXISTS movies_search CASCADE; + DROP TABLE IF EXISTS series_search CASCADE; + + DROP TABLE IF EXISTS releases_search_results CASCADE; + DROP TABLE IF EXISTS movies_search_results CASCADE; + `); +}; + +exports.down = async function(knex) { + await knex.schema.dropTable('queue'); + + await knex('users_roles') + .update('abilities', JSON.stringify([ + { subject: 'scene', action: 'create' }, + { subject: 'scene', action: 'update' }, + { subject: 'scene', action: 'delete' }, + { subject: 'actor', action: 'create' }, + { subject: 'actor', action: 'update' }, + { subject: 'actor', action: 'delete' }, + { subject: 'actor', action: 'merge' }, + { plainUrls: true }, + ])) + .where('role', 'admin'); + + await knex.schema.createTable('releases_search', (table) => { + table.integer('release_id', 16) + .references('id') + .inTable('releases') + .onDelete('cascade'); + }); + + await knex.schema.createTable('movies_search', (table) => { + table.integer('movie_id', 16) + .references('id') + .inTable('movies') + .onDelete('cascade'); + }); + + await knex.schema.createTable('series_search', (table) => { + table.integer('serie_id', 16) + .references('id') + .inTable('series') + .onDelete('cascade'); + }); + + await knex.raw(` + ALTER TABLE releases_search ADD COLUMN document tsvector; + ALTER TABLE movies_search ADD COLUMN document tsvector; + ALTER TABLE series_search ADD COLUMN document tsvector; + + CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id); + CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id); + CREATE INDEX releases_search_index ON releases_search USING GIN (document); + CREATE INDEX movies_search_index ON movies_search USING GIN (document); + CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id); + CREATE INDEX series_search_index ON series_search USING GIN (document); + `); +}; diff --git a/src/tools/update-search.js b/src/tools/update-search.js deleted file mode 100644 index 0e3a22ea..00000000 --- a/src/tools/update-search.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -const { updateSceneSearch, updateMovieSearch } = require('../update-search'); - -async function init() { - await updateSceneSearch(); - await updateMovieSearch(); - - process.exit(); -} - -init(); diff --git a/src/update-search.js b/src/update-search.js index 6cd21481..ae7a87bd 100644 --- a/src/update-search.js +++ b/src/update-search.js @@ -1,433 +1,34 @@ 'use strict'; -const manticore = require('manticoresearch'); -const { format } = require('date-fns'); +const config = require('config'); +const unprint = require('unprint'); const knex = require('./knex'); -const logger = require('./logger')(__filename); -const bulkInsert = require('./utils/bulk-insert'); -const chunk = require('./utils/chunk'); -const filterTitle = require('./utils/filter-title'); -const mantiClient = new manticore.ApiClient(); -const indexApi = new manticore.IndexApi(mantiClient); +async function syncWeb(domain, ids) { + await knex('queue').insert({ domain, item_ids: ids }); -async function updateManticoreStashedScenes(docs) { - await chunk(docs, 1000).reduce(async (chain, docsChunk) => { - await chain; - - const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id); - - const stashes = await knex('stashes_scenes') - .select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id') - .leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id') - .whereIn('scene_id', sceneIds); - - const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => { - const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id); - - if (sceneStashes.length === 0) { - return []; - } - - const stashDoc = sceneStashes.map((stash) => ({ - replace: { - index: 'scenes_stashed', - id: stash.stashed_id, - doc: { - // ...doc.replace.doc, - scene_id: doc.replace.id, - user_id: stash.user_id, - stash_id: stash.stash_id, - created_at: Math.round(stash.created_at.getTime() / 1000), - }, - }, - })); - - return stashDoc; - }); - - if (stashDocs.length > 0) { - await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n')); - } - - const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id); - - if (deleteSceneIds.length > 0) { - await indexApi.callDelete({ - index: 'scenes_stashed', - query: { - bool: { - must: [ - { - in: { - scene_id: deleteSceneIds, - }, - }, - ], - }, - }, - }); - } - }, Promise.resolve()); -} - -async function updateManticoreSceneSearch(releaseIds) { - logger.info(`Updating Manticore search documents for ${releaseIds ? releaseIds.length : 'all' } scenes`); - - const scenes = await knex.raw(` - SELECT - releases.id AS id, - releases.title, - releases.created_at, - releases.date, - releases.shoot_id, - scenes_meta.stashed, - entities.id as channel_id, - entities.slug as channel_slug, - entities.name as channel_name, - parents.id as network_id, - parents.slug as network_slug, - parents.name as network_name, - studios.id as studio_id, - studios.slug as studio_slug, - studios.name as studio_name, - grandparents.id as parent_network_id, - COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, - COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, - COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies, - COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series, - studios.showcased IS NOT false - AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true) - AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true) - AND (releases_summaries.batch_showcased IS NOT false) - AS showcased, - row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index - FROM releases - LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id - LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id - LEFT JOIN entities ON releases.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id - LEFT JOIN entities AS studios ON studios.id = releases.studio_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id - LEFT JOIN movies ON movies.id = movies_scenes.movie_id - LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id - LEFT JOIN series ON series.id = series_scenes.serie_id - ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} - GROUP BY - releases.id, - releases.title, - releases.created_at, - releases.date, - releases.shoot_id, - scenes_meta.stashed, - releases_summaries.batch_showcased, - entities.id, - entities.name, - entities.slug, - entities.alias, - entities.showcased, - parents.id, - parents.name, - parents.slug, - parents.alias, - grandparents.id, - studios.id, - studios.name, - studios.slug, - parents.showcased, - studios.showcased - `, releaseIds && [releaseIds]); - - const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene])); - - const docs = releaseIds.map((sceneId) => { - const scene = scenesById[sceneId]; - - if (!scene) { - return { - delete: { - index: 'scenes', - id: sceneId, - }, - }; - } - - const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' ')); - const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results - const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]); - - return { - replace: { - index: 'scenes', - id: scene.id, - doc: { - title: scene.title || undefined, - title_filtered: filteredTitle || undefined, - date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined, - created_at: Math.round(scene.created_at.getTime() / 1000), - effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000), - is_showcased: scene.showcased, - shoot_id: scene.shoot_id || undefined, - channel_id: scene.channel_id, - channel_slug: scene.channel_slug, - channel_name: scene.channel_name, - network_id: scene.network_id || undefined, - network_slug: scene.network_slug || undefined, - network_name: scene.network_name || undefined, - studio_id: scene.studio_id || undefined, - studio_slug: scene.studio_slug || undefined, - studio_name: scene.studio_name || undefined, - entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN - actor_ids: scene.actors.map((actor) => actor.f1), - actors: scene.actors.map((actor) => actor.f2).join(), - tag_ids: scene.tags.map((tag) => tag.f1), - tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results - movie_ids: scene.movies.map((movie) => movie.f1), - movies: scene.movies.map((movie) => movie.f2).join(' '), - serie_ids: scene.series.map((serie) => serie.f1), - series: scene.series.map((serie) => serie.f2).join(' '), - meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined, - stashed: scene.stashed || 0, - dupe_index: scene.dupe_index || 0, - }, + if (config.webApi.enabled) { + await unprint.post(`${config.webApi.address}/sync`, null, { + headers: { + 'api-user': config.webApi.apiUserId, + 'api-key': config.webApi.apiKey, }, - }; - }); - - if (docs.length === 0) { - return; + }); } - - await Promise.all([ - indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')), - updateManticoreStashedScenes(docs), - ]); -} - -async function updateSqlSceneSearch(releaseIds) { - logger.info(`Updating SQL search documents for ${releaseIds ? releaseIds.length : 'all' } releases`); - - const documents = await knex.raw(` - SELECT - releases.id AS release_id, - TO_TSVECTOR( - 'english', - COALESCE(releases.title, '') || ' ' || - releases.entry_id || ' ' || - entities.name || ' ' || - entities.slug || ' ' || - COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || - COALESCE(parents.name, '') || ' ' || - COALESCE(parents.slug, '') || ' ' || - COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || - COALESCE(releases.shoot_id, '') || ' ' || - COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || - STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags_aliases.name, ''), ' ') - ) as document - FROM releases - LEFT JOIN entities ON releases.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} - GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; - `, releaseIds && [releaseIds]); - - if (documents.rows?.length > 0) { - await bulkInsert('releases_search', documents.rows, ['release_id']); - } - - await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;'); } async function updateSceneSearch(releaseIds) { await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;'); + await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;'); - await updateSqlSceneSearch(releaseIds); - await updateManticoreSceneSearch(releaseIds); -} - -async function updateManticoreMovieSearch(movieIds) { - logger.info(`Updating Manticore search documents for ${movieIds ? movieIds.length : 'all' } movies`); - - const movies = await knex.raw(` - SELECT - movies.id AS id, - movies.title, - movies.created_at, - movies.date, - movies_meta.stashed, - entities.id as channel_id, - entities.slug as channel_slug, - entities.name as channel_name, - parents.id as network_id, - parents.slug as network_slug, - parents.name as network_name, - movies_covers IS NOT NULL as has_cover, - COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, - COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, - COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags, - row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index - FROM movies - LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id - LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id - LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id - LEFT JOIN entities ON movies.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id - LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true - LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id - ${movieIds ? 'WHERE movies.id = ANY(?)' : ''} - GROUP BY - movies.id, - movies.title, - movies.created_at, - movies.date, - movies_meta.stashed, - movies_meta.stashed_scenes, - movies_meta.stashed_total, - entities.id, - entities.name, - entities.slug, - entities.alias, - parents.id, - parents.name, - parents.slug, - parents.alias, - movies_covers.* - `, movieIds && [movieIds]); - - const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie])); - - const docs = movieIds.map((movieId) => { - const movie = moviesById[movieId]; - - if (!movie) { - return { - delete: { - index: 'movies', - id: movieId, - }, - }; - } - - const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, { - id: tag.f1, - name: tag.f2, - priority: tag.f3, - alias: tag.f4, - }]))); - - const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc. - const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results - const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' '); - - return { - replace: { - index: 'movies', - id: movie.id, - doc: { - title: movie.title || undefined, - title_filtered: filteredTitle || undefined, - date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined, - created_at: Math.round(movie.created_at.getTime() / 1000), - effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000), - channel_id: movie.channel_id, - channel_slug: movie.channel_slug, - channel_name: movie.channel_name, - network_id: movie.network_id || undefined, - network_slug: movie.network_slug || undefined, - network_name: movie.network_name || undefined, - entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN - actor_ids: movie.actors.map((actor) => actor.f1), - actors: movie.actors.map((actor) => actor.f2).join(), - tag_ids: combinedTags.map((tag) => tag.id), - tags: flatTags.join(' '), - has_cover: movie.has_cover, - meta: movie.date ? format(movie.date, 'y yy M MMM MMMM d') : undefined, - stashed: movie.stashed || 0, - stashed_scenes: movie.stashed_scenes || 0, - stashed_total: movie.stashed_total || 0, - dupe_index: movie.dupe_index || 0, - }, - }, - }; - }); - - if (docs.length === 0) { - return; - } - - await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); -} - -async function updateSqlMovieSearch(movieIds, target = 'movie') { - logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`); - - const documents = await knex.raw(` - SELECT - ${target}s.id AS ${target}_id, - TO_TSVECTOR( - 'english', - COALESCE(${target}s.title, '') || ' ' || - entities.name || ' ' || - entities.slug || ' ' || - COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || - COALESCE(parents.name, '') || ' ' || - COALESCE(parents.slug, '') || ' ' || - COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || - COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || - STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' || - STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags.name, ''), ' ') - ) as document - FROM ${target}s - LEFT JOIN entities ON ${target}s.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id - LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id - LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id - LEFT JOIN releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN actors ON actors.id = releases_actors.actor_id - LEFT JOIN tags ON tags.id = releases_tags.tag_id - ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''} - GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; - `, movieIds && [movieIds]); - - if (documents.rows?.length > 0) { - await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]); - } + await syncWeb('scene', releaseIds); } async function updateMovieSearch(releaseIds) { await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;'); - await updateSqlMovieSearch(releaseIds); - await updateManticoreMovieSearch(releaseIds); + await syncWeb('movie', releaseIds); } module.exports = {