190 lines
6.9 KiB
JavaScript
190 lines
6.9 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const manticore = require('manticoresearch');
|
|
const args = require('yargs').argv;
|
|
const { format } = require('date-fns');
|
|
|
|
const knex = require('../knex');
|
|
const filterTitle = require('../utils/filter-title');
|
|
|
|
const mantiClient = new manticore.ApiClient();
|
|
|
|
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
|
|
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
const indexApi = new manticore.IndexApi(mantiClient);
|
|
|
|
const update = args.update;
|
|
|
|
async function fetchScenes() {
|
|
const scenes = await knex.raw(`
|
|
SELECT
|
|
releases.id AS id,
|
|
releases.title,
|
|
releases.created_at,
|
|
releases.date,
|
|
releases.entry_id,
|
|
releases.shoot_id,
|
|
scenes_meta.stashed,
|
|
entities.id as channel_id,
|
|
entities.slug as channel_slug,
|
|
entities.name as channel_name,
|
|
parents.id as network_id,
|
|
parents.slug as network_slug,
|
|
parents.name as network_name,
|
|
studios.id as studio_id,
|
|
studios.slug as studio_slug,
|
|
studios.name as studio_name,
|
|
grandparents.id as parent_network_id,
|
|
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
|
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
|
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
|
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
|
studios.showcased IS NOT false
|
|
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
|
|
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
|
|
AND (releases_summaries.batch_showcased IS NOT false)
|
|
AS showcased,
|
|
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
|
|
FROM releases
|
|
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
|
|
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
|
|
LEFT JOIN entities ON releases.entity_id = entities.id
|
|
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
|
|
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
|
|
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
|
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
|
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
|
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
|
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
|
|
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
|
|
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
|
|
LEFT JOIN series ON series.id = series_scenes.serie_id
|
|
GROUP BY
|
|
releases.id,
|
|
releases.title,
|
|
releases.created_at,
|
|
releases.date,
|
|
releases.entry_id,
|
|
releases.shoot_id,
|
|
scenes_meta.stashed,
|
|
releases_summaries.batch_showcased,
|
|
entities.id,
|
|
entities.name,
|
|
entities.slug,
|
|
entities.alias,
|
|
parents.id,
|
|
parents.name,
|
|
parents.slug,
|
|
parents.alias,
|
|
grandparents.id,
|
|
studios.id,
|
|
studios.name,
|
|
studios.slug,
|
|
entities.showcased,
|
|
parents.showcased,
|
|
studios.showcased;
|
|
`);
|
|
|
|
return scenes.rows;
|
|
}
|
|
|
|
async function init() {
|
|
if (update) {
|
|
await utilsApi.sql('drop table if exists scenes');
|
|
await utilsApi.sql(`create table scenes (
|
|
id int,
|
|
title text,
|
|
title_filtered text,
|
|
entry_id text,
|
|
shoot_id text,
|
|
channel_id int,
|
|
channel_name text,
|
|
channel_slug text,
|
|
network_id int,
|
|
network_name text,
|
|
network_slug text,
|
|
studio_id int,
|
|
studio_name text,
|
|
studio_slug text,
|
|
entity_ids multi,
|
|
actor_ids multi,
|
|
actors text,
|
|
tag_ids multi,
|
|
tags text,
|
|
movie_ids multi,
|
|
movies text,
|
|
serie_ids multi,
|
|
series text,
|
|
meta text,
|
|
date timestamp,
|
|
is_showcased bool,
|
|
created_at timestamp,
|
|
effective_date timestamp,
|
|
stashed int,
|
|
dupe_index int
|
|
)`);
|
|
|
|
const scenes = await fetchScenes();
|
|
|
|
console.log(scenes.length, scenes.at(0));
|
|
|
|
const docs = scenes.map((scene) => {
|
|
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
|
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
|
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
|
|
|
return {
|
|
replace: {
|
|
index: 'scenes',
|
|
id: scene.id,
|
|
doc: {
|
|
title: scene.title || undefined,
|
|
title_filtered: filteredTitle || undefined,
|
|
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
|
created_at: Math.round(scene.created_at.getTime() / 1000),
|
|
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
|
is_showcased: scene.showcased,
|
|
entry_id: scene.entry_id || undefined,
|
|
shoot_id: scene.shoot_id || undefined,
|
|
channel_id: scene.channel_id,
|
|
channel_slug: scene.channel_slug,
|
|
channel_name: scene.channel_name,
|
|
network_id: scene.network_id || undefined,
|
|
network_slug: scene.network_slug || undefined,
|
|
network_name: scene.network_name || undefined,
|
|
studio_id: scene.studio_id || undefined,
|
|
studio_slug: scene.studio_slug || undefined,
|
|
studio_name: scene.studio_name || undefined,
|
|
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
|
actor_ids: scene.actors.map((actor) => actor.f1),
|
|
actors: scene.actors.map((actor) => actor.f2).join(),
|
|
tag_ids: scene.tags.map((tag) => tag.f1),
|
|
tags: flatTags.join(' '),
|
|
movie_ids: scene.movies.map((movie) => movie.f1),
|
|
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
|
serie_ids: scene.series.map((serie) => serie.f1),
|
|
series: scene.series.map((serie) => serie.f2).join(' '),
|
|
meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
|
|
stashed: scene.stashed || 0,
|
|
dupe_index: scene.dupe_index || 0,
|
|
},
|
|
},
|
|
};
|
|
});
|
|
|
|
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
|
|
console.log('data', data);
|
|
}
|
|
|
|
knex.destroy();
|
|
}
|
|
|
|
init();
|