'use strict'; const config = require('config'); const manticore = require('manticoresearch'); const args = require('yargs').argv; const knex = require('../knex'); const mantiClient = new manticore.ApiClient(); mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`; const searchApi = new manticore.SearchApi(mantiClient); const utilsApi = new manticore.UtilsApi(mantiClient); const indexApi = new manticore.IndexApi(mantiClient); const update = args.update; async function fetchScenes() { const scenes = await knex.raw(` SELECT scenes_meta.id AS id, scenes_meta.title, scenes_meta.created_at, scenes_meta.date, scenes_meta.entry_id, scenes_meta.stashed, entities.id as channel_id, entities.slug as channel_slug, entities.name as channel_name, parents.id as network_id, parents.slug as network_slug, parents.name as network_name, COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags FROM scenes_meta LEFT JOIN entities ON scenes_meta.entity_id = entities.id LEFT JOIN entities AS parents ON parents.id = entities.parent_id LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = scenes_meta.id LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = scenes_meta.id LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = scenes_meta.id LEFT JOIN actors ON local_actors.actor_id = actors.id LEFT JOIN actors AS directors ON local_directors.director_id = directors.id LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true GROUP BY scenes_meta.id, scenes_meta.title, scenes_meta.created_at, scenes_meta.date, scenes_meta.entry_id, scenes_meta.stashed, entities.id, entities.name, entities.slug, entities.alias, parents.id, parents.name, parents.slug, parents.alias; `); return scenes.rows; } async function init() { if (update) { await utilsApi.sql('drop table scenes'); await utilsApi.sql(`create table scenes ( id int, title text, entry_id text, channel_id int, channel_name text, channel_slug text, network_id int, network_name text, network_slug text, actor_ids multi, actors text, tag_ids multi, tags text, date timestamp, created_at timestamp, effective_date timestamp, stashed int )`); const scenes = await fetchScenes(); const docs = scenes.map((scene) => ({ replace: { index: 'scenes', id: scene.id, doc: { title: scene.title || undefined, date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined, created_at: Math.round(scene.created_at.getTime() / 1000), effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000), entry_id: scene.entry_id, channel_id: scene.channel_id, channel_slug: scene.channel_slug, channel_name: scene.channel_name, network_id: scene.network_id || undefined, network_slug: scene.network_slug || undefined, network_name: scene.network_name || undefined, actor_ids: scene.actors.map((actor) => actor.f1), actors: scene.actors.map((actor) => actor.f2).join(), tag_ids: scene.tags.map((tag) => tag.f1), tags: scene.tags.map((tag) => tag.f2).join(), stashed: scene.stashed || 0, }, }, })); const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); console.log('data', data); } /* const searchRequest = new manticore.SearchRequest(); searchRequest.index = 'scenes'; searchRequest.attr_filter = new manticore.InFilter('ratings', [600]); const result = await searchApi.search(searchRequest); */ const result = await searchApi.search({ index: 'scenes', query: { bool: { must: [ /* { match: { '*': 'going-all-out-with-a-gangbang_1080p.mp4', }, }, */ { range: { effective_date: { lte: Math.round(Date.now() / 1000), }, }, }, /* { equals: { actor_ids: 81930, }, }, */ ], must_not: [ { in: { tag_ids: [101, 180, 32], }, }, ], }, }, limit: 30, sort: [{ effective_date: 'desc' }], /* aggs: { tags: { terms: { field: 'tag_ids', size: 100, }, }, actors: { terms: { field: 'actor_ids', size: 100, }, }, }, */ }); console.log(result); /* if (result.hits) { result.hits.hits.map((hit) => console.log(hit)); // console.log(result.aggregations.actors); // console.log(result.aggregations.tags); const sceneIds = result.hits.hits.map((hit) => hit._id); console.time('fetch'); const [scenes, actors, tags, posters, photos] = await Promise.all([ knex('releases').whereIn('id', sceneIds), knex('releases_actors') .whereIn('release_id', sceneIds) .leftJoin('actors', 'actors.id', 'releases_actors.actor_id'), knex('releases_tags') .whereIn('release_id', sceneIds) .leftJoin('tags', 'tags.id', 'releases_tags.tag_id'), knex('releases_posters') .whereIn('release_id', sceneIds) .leftJoin('media', 'media.id', 'releases_posters.media_id'), knex('releases_photos') .whereIn('release_id', sceneIds) .leftJoin('media', 'media.id', 'releases_photos.media_id'), ]); console.timeEnd('fetch'); console.time('join'); const joinedScenes = await knex('releases') .select(knex.raw(` releases.*, COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, COALESCE(json_agg(DISTINCT posters) FILTER (WHERE posters.id IS NOT NULL), '[]') as posters, COALESCE(json_agg(DISTINCT photos) FILTER (WHERE photos.id IS NOT NULL), '[]') as photos `)) .whereIn('releases.id', sceneIds) .leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id') .leftJoin('actors', 'actors.id', 'releases_actors.actor_id') .leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id') .leftJoin('tags', 'tags.id', 'releases_tags.tag_id') .leftJoin('releases_posters', 'releases_posters.release_id', 'releases.id') .leftJoin('media as posters', 'posters.id', 'releases_posters.media_id') .leftJoin('releases_photos', 'releases_photos.release_id', 'releases.id') .leftJoin('media as photos', 'photos.id', 'releases_photos.media_id') .groupBy('releases.id'); // console.log(joinedScenes[0]); console.timeEnd('join'); console.log(sceneIds); console.log(scenes); console.log(actors); console.log(tags); console.log(posters); console.log(photos); } */ knex.destroy(); } init();