Replaced direct Manticore update with queue and web server ping.
This commit is contained in:
DebaucheryLibrarian
@@ -199,6 +199,12 @@ module.exports = {
|
|||||||
// source: 'http://nsfw.unknown.name/random',
|
// source: 'http://nsfw.unknown.name/random',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
webApi: {
|
||||||
|
enabled: true,
|
||||||
|
address: 'http://localhost:5100/api',
|
||||||
|
apiUserId: 1,
|
||||||
|
apiKey: null,
|
||||||
|
},
|
||||||
proxy: {
|
proxy: {
|
||||||
enable: false,
|
enable: false,
|
||||||
test: 'https://api.ipify.org?format=json',
|
test: 'https://api.ipify.org?format=json',
|
||||||
|
|||||||
87
migrations/20260608053154_sync_abilities.js
Normal file
87
migrations/20260608053154_sync_abilities.js
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
exports.up = async function(knex) {
|
||||||
|
await knex.schema.createTable('queue', (table) => {
|
||||||
|
table.increments('id');
|
||||||
|
|
||||||
|
table.string('domain');
|
||||||
|
table.specificType('item_ids', 'integer array');
|
||||||
|
|
||||||
|
table.text('comment');
|
||||||
|
|
||||||
|
table.datetime('created_at')
|
||||||
|
.defaultTo(knex.fn.now());
|
||||||
|
});
|
||||||
|
|
||||||
|
await knex('users_roles')
|
||||||
|
.update('abilities', JSON.stringify([
|
||||||
|
{ subject: 'scene', action: 'create' },
|
||||||
|
{ subject: 'scene', action: 'update' },
|
||||||
|
{ subject: 'scene', action: 'delete' },
|
||||||
|
{ subject: 'actor', action: 'create' },
|
||||||
|
{ subject: 'actor', action: 'update' },
|
||||||
|
{ subject: 'actor', action: 'delete' },
|
||||||
|
{ subject: 'actor', action: 'merge' },
|
||||||
|
{ subject: 'sync' },
|
||||||
|
{ subject: 'plainUrls' },
|
||||||
|
]))
|
||||||
|
.where('role', 'admin');
|
||||||
|
|
||||||
|
await knex.raw(`
|
||||||
|
DROP TABLE IF EXISTS releases_search CASCADE;
|
||||||
|
DROP TABLE IF EXISTS movies_search CASCADE;
|
||||||
|
DROP TABLE IF EXISTS series_search CASCADE;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS releases_search_results CASCADE;
|
||||||
|
DROP TABLE IF EXISTS movies_search_results CASCADE;
|
||||||
|
`);
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.down = async function(knex) {
|
||||||
|
await knex.schema.dropTable('queue');
|
||||||
|
|
||||||
|
await knex('users_roles')
|
||||||
|
.update('abilities', JSON.stringify([
|
||||||
|
{ subject: 'scene', action: 'create' },
|
||||||
|
{ subject: 'scene', action: 'update' },
|
||||||
|
{ subject: 'scene', action: 'delete' },
|
||||||
|
{ subject: 'actor', action: 'create' },
|
||||||
|
{ subject: 'actor', action: 'update' },
|
||||||
|
{ subject: 'actor', action: 'delete' },
|
||||||
|
{ subject: 'actor', action: 'merge' },
|
||||||
|
{ plainUrls: true },
|
||||||
|
]))
|
||||||
|
.where('role', 'admin');
|
||||||
|
|
||||||
|
await knex.schema.createTable('releases_search', (table) => {
|
||||||
|
table.integer('release_id', 16)
|
||||||
|
.references('id')
|
||||||
|
.inTable('releases')
|
||||||
|
.onDelete('cascade');
|
||||||
|
});
|
||||||
|
|
||||||
|
await knex.schema.createTable('movies_search', (table) => {
|
||||||
|
table.integer('movie_id', 16)
|
||||||
|
.references('id')
|
||||||
|
.inTable('movies')
|
||||||
|
.onDelete('cascade');
|
||||||
|
});
|
||||||
|
|
||||||
|
await knex.schema.createTable('series_search', (table) => {
|
||||||
|
table.integer('serie_id', 16)
|
||||||
|
.references('id')
|
||||||
|
.inTable('series')
|
||||||
|
.onDelete('cascade');
|
||||||
|
});
|
||||||
|
|
||||||
|
await knex.raw(`
|
||||||
|
ALTER TABLE releases_search ADD COLUMN document tsvector;
|
||||||
|
ALTER TABLE movies_search ADD COLUMN document tsvector;
|
||||||
|
ALTER TABLE series_search ADD COLUMN document tsvector;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
|
||||||
|
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
|
||||||
|
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
|
||||||
|
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
|
||||||
|
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
|
||||||
|
CREATE INDEX series_search_index ON series_search USING GIN (document);
|
||||||
|
`);
|
||||||
|
};
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const { updateSceneSearch, updateMovieSearch } = require('../update-search');
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
await updateSceneSearch();
|
|
||||||
await updateMovieSearch();
|
|
||||||
|
|
||||||
process.exit();
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
@@ -1,433 +1,34 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const manticore = require('manticoresearch');
|
const config = require('config');
|
||||||
const { format } = require('date-fns');
|
const unprint = require('unprint');
|
||||||
|
|
||||||
const knex = require('./knex');
|
const knex = require('./knex');
|
||||||
const logger = require('./logger')(__filename);
|
|
||||||
const bulkInsert = require('./utils/bulk-insert');
|
|
||||||
const chunk = require('./utils/chunk');
|
|
||||||
const filterTitle = require('./utils/filter-title');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
async function syncWeb(domain, ids) {
|
||||||
const indexApi = new manticore.IndexApi(mantiClient);
|
await knex('queue').insert({ domain, item_ids: ids });
|
||||||
|
|
||||||
async function updateManticoreStashedScenes(docs) {
|
if (config.webApi.enabled) {
|
||||||
await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
|
await unprint.post(`${config.webApi.address}/sync`, null, {
|
||||||
await chain;
|
headers: {
|
||||||
|
'api-user': config.webApi.apiUserId,
|
||||||
const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
|
'api-key': config.webApi.apiKey,
|
||||||
|
|
||||||
const stashes = await knex('stashes_scenes')
|
|
||||||
.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
|
|
||||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
|
||||||
.whereIn('scene_id', sceneIds);
|
|
||||||
|
|
||||||
const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
|
|
||||||
const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
|
|
||||||
|
|
||||||
if (sceneStashes.length === 0) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
const stashDoc = sceneStashes.map((stash) => ({
|
|
||||||
replace: {
|
|
||||||
index: 'scenes_stashed',
|
|
||||||
id: stash.stashed_id,
|
|
||||||
doc: {
|
|
||||||
// ...doc.replace.doc,
|
|
||||||
scene_id: doc.replace.id,
|
|
||||||
user_id: stash.user_id,
|
|
||||||
stash_id: stash.stash_id,
|
|
||||||
created_at: Math.round(stash.created_at.getTime() / 1000),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
return stashDoc;
|
|
||||||
});
|
|
||||||
|
|
||||||
if (stashDocs.length > 0) {
|
|
||||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
||||||
}
|
|
||||||
|
|
||||||
const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
|
|
||||||
|
|
||||||
if (deleteSceneIds.length > 0) {
|
|
||||||
await indexApi.callDelete({
|
|
||||||
index: 'scenes_stashed',
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{
|
|
||||||
in: {
|
|
||||||
scene_id: deleteSceneIds,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}, Promise.resolve());
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateManticoreSceneSearch(releaseIds) {
|
|
||||||
logger.info(`Updating Manticore search documents for ${releaseIds ? releaseIds.length : 'all' } scenes`);
|
|
||||||
|
|
||||||
const scenes = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
releases.id AS id,
|
|
||||||
releases.title,
|
|
||||||
releases.created_at,
|
|
||||||
releases.date,
|
|
||||||
releases.shoot_id,
|
|
||||||
scenes_meta.stashed,
|
|
||||||
entities.id as channel_id,
|
|
||||||
entities.slug as channel_slug,
|
|
||||||
entities.name as channel_name,
|
|
||||||
parents.id as network_id,
|
|
||||||
parents.slug as network_slug,
|
|
||||||
parents.name as network_name,
|
|
||||||
studios.id as studio_id,
|
|
||||||
studios.slug as studio_slug,
|
|
||||||
studios.name as studio_name,
|
|
||||||
grandparents.id as parent_network_id,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
|
||||||
studios.showcased IS NOT false
|
|
||||||
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
|
|
||||||
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
|
|
||||||
AND (releases_summaries.batch_showcased IS NOT false)
|
|
||||||
AS showcased,
|
|
||||||
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
|
|
||||||
FROM releases
|
|
||||||
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
|
|
||||||
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
|
|
||||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
|
|
||||||
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
|
|
||||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
|
||||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
||||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
||||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
|
||||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
||||||
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
|
|
||||||
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
|
|
||||||
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
|
|
||||||
LEFT JOIN series ON series.id = series_scenes.serie_id
|
|
||||||
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
|
||||||
GROUP BY
|
|
||||||
releases.id,
|
|
||||||
releases.title,
|
|
||||||
releases.created_at,
|
|
||||||
releases.date,
|
|
||||||
releases.shoot_id,
|
|
||||||
scenes_meta.stashed,
|
|
||||||
releases_summaries.batch_showcased,
|
|
||||||
entities.id,
|
|
||||||
entities.name,
|
|
||||||
entities.slug,
|
|
||||||
entities.alias,
|
|
||||||
entities.showcased,
|
|
||||||
parents.id,
|
|
||||||
parents.name,
|
|
||||||
parents.slug,
|
|
||||||
parents.alias,
|
|
||||||
grandparents.id,
|
|
||||||
studios.id,
|
|
||||||
studios.name,
|
|
||||||
studios.slug,
|
|
||||||
parents.showcased,
|
|
||||||
studios.showcased
|
|
||||||
`, releaseIds && [releaseIds]);
|
|
||||||
|
|
||||||
const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
|
|
||||||
|
|
||||||
const docs = releaseIds.map((sceneId) => {
|
|
||||||
const scene = scenesById[sceneId];
|
|
||||||
|
|
||||||
if (!scene) {
|
|
||||||
return {
|
|
||||||
delete: {
|
|
||||||
index: 'scenes',
|
|
||||||
id: sceneId,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
|
|
||||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
|
|
||||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
|
||||||
|
|
||||||
return {
|
|
||||||
replace: {
|
|
||||||
index: 'scenes',
|
|
||||||
id: scene.id,
|
|
||||||
doc: {
|
|
||||||
title: scene.title || undefined,
|
|
||||||
title_filtered: filteredTitle || undefined,
|
|
||||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
|
||||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
|
||||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
|
||||||
is_showcased: scene.showcased,
|
|
||||||
shoot_id: scene.shoot_id || undefined,
|
|
||||||
channel_id: scene.channel_id,
|
|
||||||
channel_slug: scene.channel_slug,
|
|
||||||
channel_name: scene.channel_name,
|
|
||||||
network_id: scene.network_id || undefined,
|
|
||||||
network_slug: scene.network_slug || undefined,
|
|
||||||
network_name: scene.network_name || undefined,
|
|
||||||
studio_id: scene.studio_id || undefined,
|
|
||||||
studio_slug: scene.studio_slug || undefined,
|
|
||||||
studio_name: scene.studio_name || undefined,
|
|
||||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
|
||||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
|
||||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
|
||||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
|
||||||
tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
|
|
||||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
|
||||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
|
||||||
serie_ids: scene.series.map((serie) => serie.f1),
|
|
||||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
|
||||||
meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
|
|
||||||
stashed: scene.stashed || 0,
|
|
||||||
dupe_index: scene.dupe_index || 0,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
};
|
});
|
||||||
});
|
|
||||||
|
|
||||||
if (docs.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await Promise.all([
|
|
||||||
indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')),
|
|
||||||
updateManticoreStashedScenes(docs),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateSqlSceneSearch(releaseIds) {
|
|
||||||
logger.info(`Updating SQL search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
|
|
||||||
|
|
||||||
const documents = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
releases.id AS release_id,
|
|
||||||
TO_TSVECTOR(
|
|
||||||
'english',
|
|
||||||
COALESCE(releases.title, '') || ' ' ||
|
|
||||||
releases.entry_id || ' ' ||
|
|
||||||
entities.name || ' ' ||
|
|
||||||
entities.slug || ' ' ||
|
|
||||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
|
||||||
COALESCE(parents.name, '') || ' ' ||
|
|
||||||
COALESCE(parents.slug, '') || ' ' ||
|
|
||||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
|
||||||
COALESCE(releases.shoot_id, '') || ' ' ||
|
|
||||||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
|
|
||||||
) as document
|
|
||||||
FROM releases
|
|
||||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
|
||||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
||||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
||||||
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
|
|
||||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
||||||
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
|
||||||
GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
|
||||||
`, releaseIds && [releaseIds]);
|
|
||||||
|
|
||||||
if (documents.rows?.length > 0) {
|
|
||||||
await bulkInsert('releases_search', documents.rows, ['release_id']);
|
|
||||||
}
|
|
||||||
|
|
||||||
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updateSceneSearch(releaseIds) {
|
async function updateSceneSearch(releaseIds) {
|
||||||
await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
|
await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
|
||||||
|
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
|
||||||
|
|
||||||
await updateSqlSceneSearch(releaseIds);
|
await syncWeb('scene', releaseIds);
|
||||||
await updateManticoreSceneSearch(releaseIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateManticoreMovieSearch(movieIds) {
|
|
||||||
logger.info(`Updating Manticore search documents for ${movieIds ? movieIds.length : 'all' } movies`);
|
|
||||||
|
|
||||||
const movies = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
movies.id AS id,
|
|
||||||
movies.title,
|
|
||||||
movies.created_at,
|
|
||||||
movies.date,
|
|
||||||
movies_meta.stashed,
|
|
||||||
entities.id as channel_id,
|
|
||||||
entities.slug as channel_slug,
|
|
||||||
entities.name as channel_name,
|
|
||||||
parents.id as network_id,
|
|
||||||
parents.slug as network_slug,
|
|
||||||
parents.name as network_name,
|
|
||||||
movies_covers IS NOT NULL as has_cover,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
|
|
||||||
row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
|
|
||||||
FROM movies
|
|
||||||
LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
|
|
||||||
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
|
|
||||||
LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
|
|
||||||
LEFT JOIN entities ON movies.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
||||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
||||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
|
||||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
||||||
LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
|
|
||||||
LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
|
|
||||||
LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
|
|
||||||
${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
|
|
||||||
GROUP BY
|
|
||||||
movies.id,
|
|
||||||
movies.title,
|
|
||||||
movies.created_at,
|
|
||||||
movies.date,
|
|
||||||
movies_meta.stashed,
|
|
||||||
movies_meta.stashed_scenes,
|
|
||||||
movies_meta.stashed_total,
|
|
||||||
entities.id,
|
|
||||||
entities.name,
|
|
||||||
entities.slug,
|
|
||||||
entities.alias,
|
|
||||||
parents.id,
|
|
||||||
parents.name,
|
|
||||||
parents.slug,
|
|
||||||
parents.alias,
|
|
||||||
movies_covers.*
|
|
||||||
`, movieIds && [movieIds]);
|
|
||||||
|
|
||||||
const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
|
|
||||||
|
|
||||||
const docs = movieIds.map((movieId) => {
|
|
||||||
const movie = moviesById[movieId];
|
|
||||||
|
|
||||||
if (!movie) {
|
|
||||||
return {
|
|
||||||
delete: {
|
|
||||||
index: 'movies',
|
|
||||||
id: movieId,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
|
|
||||||
id: tag.f1,
|
|
||||||
name: tag.f2,
|
|
||||||
priority: tag.f3,
|
|
||||||
alias: tag.f4,
|
|
||||||
}])));
|
|
||||||
|
|
||||||
const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
|
||||||
const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
|
||||||
const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
|
|
||||||
|
|
||||||
return {
|
|
||||||
replace: {
|
|
||||||
index: 'movies',
|
|
||||||
id: movie.id,
|
|
||||||
doc: {
|
|
||||||
title: movie.title || undefined,
|
|
||||||
title_filtered: filteredTitle || undefined,
|
|
||||||
date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
|
|
||||||
created_at: Math.round(movie.created_at.getTime() / 1000),
|
|
||||||
effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
|
|
||||||
channel_id: movie.channel_id,
|
|
||||||
channel_slug: movie.channel_slug,
|
|
||||||
channel_name: movie.channel_name,
|
|
||||||
network_id: movie.network_id || undefined,
|
|
||||||
network_slug: movie.network_slug || undefined,
|
|
||||||
network_name: movie.network_name || undefined,
|
|
||||||
entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
|
|
||||||
actor_ids: movie.actors.map((actor) => actor.f1),
|
|
||||||
actors: movie.actors.map((actor) => actor.f2).join(),
|
|
||||||
tag_ids: combinedTags.map((tag) => tag.id),
|
|
||||||
tags: flatTags.join(' '),
|
|
||||||
has_cover: movie.has_cover,
|
|
||||||
meta: movie.date ? format(movie.date, 'y yy M MMM MMMM d') : undefined,
|
|
||||||
stashed: movie.stashed || 0,
|
|
||||||
stashed_scenes: movie.stashed_scenes || 0,
|
|
||||||
stashed_total: movie.stashed_total || 0,
|
|
||||||
dupe_index: movie.dupe_index || 0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
if (docs.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateSqlMovieSearch(movieIds, target = 'movie') {
|
|
||||||
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
|
|
||||||
|
|
||||||
const documents = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
${target}s.id AS ${target}_id,
|
|
||||||
TO_TSVECTOR(
|
|
||||||
'english',
|
|
||||||
COALESCE(${target}s.title, '') || ' ' ||
|
|
||||||
entities.name || ' ' ||
|
|
||||||
entities.slug || ' ' ||
|
|
||||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
|
||||||
COALESCE(parents.name, '') || ' ' ||
|
|
||||||
COALESCE(parents.slug, '') || ' ' ||
|
|
||||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
|
||||||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
|
||||||
STRING_AGG(COALESCE(tags.name, ''), ' ')
|
|
||||||
) as document
|
|
||||||
FROM ${target}s
|
|
||||||
LEFT JOIN entities ON ${target}s.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
|
|
||||||
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
|
|
||||||
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
|
|
||||||
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
|
|
||||||
LEFT JOIN actors ON actors.id = releases_actors.actor_id
|
|
||||||
LEFT JOIN tags ON tags.id = releases_tags.tag_id
|
|
||||||
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
|
|
||||||
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
|
||||||
`, movieIds && [movieIds]);
|
|
||||||
|
|
||||||
if (documents.rows?.length > 0) {
|
|
||||||
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updateMovieSearch(releaseIds) {
|
async function updateMovieSearch(releaseIds) {
|
||||||
await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;');
|
await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;');
|
||||||
|
|
||||||
await updateSqlMovieSearch(releaseIds);
|
await syncWeb('movie', releaseIds);
|
||||||
await updateManticoreMovieSearch(releaseIds);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|||||||
Reference in New Issue
Block a user