traxxx/src/update-search.js

'use strict';

const manticore = require('manticoresearch');

const knex = require('./knex');
const logger = require('./logger')(__filename);
const bulkInsert = require('./utils/bulk-insert');

const mantiClient = new manticore.ApiClient();
const indexApi = new manticore.IndexApi(mantiClient);

async function updateManticoreSearch(releaseIds) {
	const scenes = await knex.raw(`
        SELECT
            scenes_meta.id AS id,
			scenes_meta.title,
			scenes_meta.created_at,
			scenes_meta.date,
			scenes_meta.entry_id,
			scenes_meta.stashed,
			entities.id as channel_id,
			entities.slug as channel_slug,
			entities.name as channel_name,
			parents.id as network_id,
			parents.slug as network_slug,
			parents.name as network_name,
			COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
			COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
        FROM scenes_meta
        LEFT JOIN entities ON scenes_meta.entity_id = entities.id
        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id
        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id
        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id
        LEFT JOIN actors ON local_actors.actor_id = actors.id
        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
        LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
        ${releaseIds ? 'WHERE scenes_meta.id = ANY(?)' : ''}
        GROUP BY
            scenes_meta.id,
			scenes_meta.title,
			scenes_meta.created_at,
			scenes_meta.date,
			scenes_meta.entry_id,
			scenes_meta.stashed,
			entities.id,
			entities.name,
			entities.slug,
			entities.alias,
			parents.id,
			parents.name,
			parents.slug,
			parents.alias;
    `, releaseIds && [releaseIds]);

	console.log(releaseIds);
	console.log(scenes);

	const docs = scenes.rows.map((scene) => ({
		replace: {
			index: 'scenes',
			id: scene.id,
			doc: {
				title: scene.title || undefined,
				date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
				created_at: Math.round(scene.created_at.getTime() / 1000),
				effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
				entry_id: scene.entry_id,
				channel_id: scene.channel_id,
				channel_slug: scene.channel_slug,
				channel_name: scene.channel_name,
				network_id: scene.network_id || undefined,
				network_slug: scene.network_slug || undefined,
				network_name: scene.network_name || undefined,
				actor_ids: scene.actors.map((actor) => actor.f1),
				actors: scene.actors.map((actor) => actor.f2).join(),
				tag_ids: scene.tags.map((tag) => tag.f1),
				tags: scene.tags.map((tag) => tag.f2).join(),
				stashed: scene.stashed || 0,
			},
		},
	}));

	console.log('docs', docs);

	if (docs.length === 0) {
		return;
	}

	const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));

	console.log('data', data);
}

async function updateSqlSearch(releaseIds) {
	logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);

	const documents = await knex.raw(`
        SELECT
            releases.id AS release_id,
            TO_TSVECTOR(
                'english',
                COALESCE(releases.title, '') || ' ' ||
                releases.entry_id || ' ' ||
                entities.name || ' ' ||
                entities.slug || ' ' ||
                COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
                COALESCE(parents.name, '') || ' ' ||
                COALESCE(parents.slug, '') || ' ' ||
                COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
                COALESCE(releases.shoot_id, '') || ' ' ||
                COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
                STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
                STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
                STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
                STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
            ) as document
        FROM releases
        LEFT JOIN entities ON releases.entity_id = entities.id
        LEFT JOIN entities AS parents ON parents.id = entities.parent_id
        LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
        LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
        LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
        LEFT JOIN actors ON local_actors.actor_id = actors.id
        LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
        LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
        LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
        ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
        GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
    `, releaseIds && [releaseIds]);

	if (documents.rows?.length > 0) {
		await bulkInsert('releases_search', documents.rows, ['release_id']);
	}

	await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
}

async function updateSceneSearch(releaseIds) {
	await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');

	await updateSqlSearch(releaseIds);
	await updateManticoreSearch(releaseIds);
}

async function updateMovieSearch(movieIds, target = 'movie') {
	logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);

	const documents = await knex.raw(`
		SELECT
			${target}s.id AS ${target}_id,
				TO_TSVECTOR(
				'english',
					COALESCE(${target}s.title, '') || ' ' ||
					entities.name || ' ' ||
					entities.slug || ' ' ||
					COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
					COALESCE(parents.name, '') || ' ' ||
					COALESCE(parents.slug, '') || ' ' ||
					COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
					COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
					STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
					STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
					STRING_AGG(COALESCE(tags.name, ''), ' ')
			) as document
		FROM ${target}s
		LEFT JOIN entities ON ${target}s.entity_id = entities.id
		LEFT JOIN entities AS parents ON parents.id = entities.parent_id
		LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
		LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
		LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
		LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
		LEFT JOIN actors ON actors.id = releases_actors.actor_id
		LEFT JOIN tags ON tags.id = releases_tags.tag_id
        ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
		GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
    `, movieIds && [movieIds]);

	if (documents.rows?.length > 0) {
		await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
	}
}

module.exports = {
	updateSceneSearch,
	updateMovieSearch,
};