Removed Manticore sync tools, use traxxx-web repo.
This commit is contained in:
DebaucheryLibrarian
@@ -647,6 +647,7 @@ const tags = [
|
|||||||
{
|
{
|
||||||
name: 'MFF threesome',
|
name: 'MFF threesome',
|
||||||
slug: 'mff',
|
slug: 'mff',
|
||||||
|
implies: ['threesome'],
|
||||||
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
|
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
|
||||||
group: 'group',
|
group: 'group',
|
||||||
},
|
},
|
||||||
@@ -830,6 +831,7 @@ const tags = [
|
|||||||
{
|
{
|
||||||
name: 'MFM threesome',
|
name: 'MFM threesome',
|
||||||
slug: 'mfm',
|
slug: 'mfm',
|
||||||
|
implies: ['threesome'],
|
||||||
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
|
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
|
||||||
group: 'group',
|
group: 'group',
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,109 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
const args = require('yargs').argv;
|
|
||||||
|
|
||||||
const knex = require('../knex');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
const searchApi = new manticore.SearchApi(mantiClient);
|
|
||||||
|
|
||||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
||||||
const indexApi = new manticore.IndexApi(mantiClient);
|
|
||||||
|
|
||||||
const update = args.update;
|
|
||||||
|
|
||||||
async function fetchActors() {
|
|
||||||
// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
|
|
||||||
const actors = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
actors.*,
|
|
||||||
actors_meta.*,
|
|
||||||
date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
|
|
||||||
FROM actors
|
|
||||||
LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id
|
|
||||||
`);
|
|
||||||
|
|
||||||
return actors.rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
if (update) {
|
|
||||||
await utilsApi.sql('drop table if exists actors');
|
|
||||||
await utilsApi.sql(`create table actors(
|
|
||||||
id int,
|
|
||||||
name text,
|
|
||||||
slug string,
|
|
||||||
entity_id int,
|
|
||||||
gender string,
|
|
||||||
date_of_birth timestamp,
|
|
||||||
country string,
|
|
||||||
has_avatar bool,
|
|
||||||
mass int,
|
|
||||||
height int,
|
|
||||||
cup string,
|
|
||||||
natural_boobs int,
|
|
||||||
penis_length int,
|
|
||||||
penis_girth int,
|
|
||||||
stashed int,
|
|
||||||
scenes int
|
|
||||||
) min_prefix_len = '3'`);
|
|
||||||
|
|
||||||
const actors = await fetchActors();
|
|
||||||
|
|
||||||
const docs = actors.map((actor) => ({
|
|
||||||
insert: {
|
|
||||||
index: 'actors',
|
|
||||||
id: actor.id,
|
|
||||||
doc: {
|
|
||||||
entity_id: actor.entity_id,
|
|
||||||
name: actor.name,
|
|
||||||
slug: actor.slug,
|
|
||||||
gender: actor.gender || undefined,
|
|
||||||
date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
|
|
||||||
has_avatar: !!actor.avatar_media_id,
|
|
||||||
country: actor.birth_country_alpha2 || undefined,
|
|
||||||
height: actor.height || undefined,
|
|
||||||
mass: actor.weight || undefined, // weight is a reserved keyword in manticore
|
|
||||||
cup: actor.cup || undefined,
|
|
||||||
natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
|
|
||||||
penis_length: actor.penis_length || undefined,
|
|
||||||
penis_girth: actor.penis_girth || undefined,
|
|
||||||
stashed: actor.stashed || 0,
|
|
||||||
scenes: actor.scenes || 0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
|
|
||||||
console.log(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log('data', data);
|
|
||||||
knex.destroy();
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await searchApi.search({
|
|
||||||
index: 'actors',
|
|
||||||
query: {
|
|
||||||
equals: {
|
|
||||||
has_avatar: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
limit: 3,
|
|
||||||
sort: [{ slug: 'asc' }],
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(result);
|
|
||||||
console.log(result.hits?.hits);
|
|
||||||
|
|
||||||
knex.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
const args = require('yargs').argv;
|
|
||||||
const { format } = require('date-fns');
|
|
||||||
|
|
||||||
const knex = require('../knex');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
// const searchApi = new manticore.SearchApi(mantiClient);
|
|
||||||
|
|
||||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
||||||
const indexApi = new manticore.IndexApi(mantiClient);
|
|
||||||
|
|
||||||
const update = args.update;
|
|
||||||
|
|
||||||
async function fetchMovies() {
|
|
||||||
const movies = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
movies.id AS id,
|
|
||||||
movies.title,
|
|
||||||
movies.created_at,
|
|
||||||
movies.date,
|
|
||||||
movies_meta.stashed,
|
|
||||||
entities.id as channel_id,
|
|
||||||
entities.slug as channel_slug,
|
|
||||||
entities.name as channel_name,
|
|
||||||
parents.id as network_id,
|
|
||||||
parents.slug as network_slug,
|
|
||||||
parents.name as network_name,
|
|
||||||
movies_covers IS NOT NULL as has_cover,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
|
|
||||||
row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
|
|
||||||
FROM movies
|
|
||||||
LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
|
|
||||||
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
|
|
||||||
LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
|
|
||||||
LEFT JOIN entities ON movies.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
|
|
||||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
||||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
||||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
|
||||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
||||||
LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
|
|
||||||
LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
|
|
||||||
LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
|
|
||||||
GROUP BY
|
|
||||||
movies.id,
|
|
||||||
movies.title,
|
|
||||||
movies.created_at,
|
|
||||||
movies.date,
|
|
||||||
movies_meta.stashed,
|
|
||||||
movies_meta.stashed_scenes,
|
|
||||||
movies_meta.stashed_total,
|
|
||||||
entities.id,
|
|
||||||
entities.name,
|
|
||||||
entities.slug,
|
|
||||||
entities.alias,
|
|
||||||
parents.id,
|
|
||||||
parents.name,
|
|
||||||
parents.slug,
|
|
||||||
parents.alias,
|
|
||||||
movies_covers.*
|
|
||||||
`);
|
|
||||||
|
|
||||||
return movies.rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
if (update) {
|
|
||||||
await utilsApi.sql('drop table if exists movies');
|
|
||||||
await utilsApi.sql(`create table movies (
|
|
||||||
id int,
|
|
||||||
title text,
|
|
||||||
title_filtered text,
|
|
||||||
channel_id int,
|
|
||||||
channel_name text,
|
|
||||||
channel_slug text,
|
|
||||||
network_id int,
|
|
||||||
network_name text,
|
|
||||||
network_slug text,
|
|
||||||
entity_ids multi,
|
|
||||||
actor_ids multi,
|
|
||||||
actors text,
|
|
||||||
tag_ids multi,
|
|
||||||
tags text,
|
|
||||||
meta text,
|
|
||||||
date timestamp,
|
|
||||||
has_cover bool,
|
|
||||||
created_at timestamp,
|
|
||||||
effective_date timestamp,
|
|
||||||
stashed int,
|
|
||||||
stashed_scenes int,
|
|
||||||
stashed_total int,
|
|
||||||
dupe_index int
|
|
||||||
)`);
|
|
||||||
|
|
||||||
const movies = await fetchMovies();
|
|
||||||
|
|
||||||
console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index));
|
|
||||||
|
|
||||||
const docs = movies.map((movie) => {
|
|
||||||
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
|
|
||||||
id: tag.f1,
|
|
||||||
name: tag.f2,
|
|
||||||
priority: tag.f3,
|
|
||||||
alias: tag.f4,
|
|
||||||
}])));
|
|
||||||
|
|
||||||
const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
|
||||||
const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
|
||||||
const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
|
|
||||||
|
|
||||||
return {
|
|
||||||
replace: {
|
|
||||||
index: 'movies',
|
|
||||||
id: movie.id,
|
|
||||||
doc: {
|
|
||||||
title: movie.title || undefined,
|
|
||||||
title_filtered: filteredTitle || undefined,
|
|
||||||
date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
|
|
||||||
created_at: Math.round(movie.created_at.getTime() / 1000),
|
|
||||||
effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
|
|
||||||
channel_id: movie.channel_id,
|
|
||||||
channel_slug: movie.channel_slug,
|
|
||||||
channel_name: movie.channel_name,
|
|
||||||
network_id: movie.network_id || undefined,
|
|
||||||
network_slug: movie.network_slug || undefined,
|
|
||||||
network_name: movie.network_name || undefined,
|
|
||||||
entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
|
|
||||||
actor_ids: movie.actors.map((actor) => actor.f1),
|
|
||||||
actors: movie.actors.map((actor) => actor.f2).join(),
|
|
||||||
tag_ids: combinedTags.map((tag) => tag.id),
|
|
||||||
tags: flatTags.join(' '),
|
|
||||||
has_cover: movie.has_cover,
|
|
||||||
meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
|
||||||
stashed: movie.stashed || 0,
|
|
||||||
stashed_scenes: movie.stashed_scenes || 0,
|
|
||||||
stashed_total: movie.stashed_total || 0,
|
|
||||||
dupe_index: movie.dupe_index || 0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(docs.map((doc) => doc.replace));
|
|
||||||
|
|
||||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
||||||
|
|
||||||
console.log('data', data);
|
|
||||||
}
|
|
||||||
|
|
||||||
knex.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
@@ -1,228 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
const args = require('yargs').argv;
|
|
||||||
const { format } = require('date-fns');
|
|
||||||
|
|
||||||
const knex = require('../knex');
|
|
||||||
const chunk = require('../utils/chunk');
|
|
||||||
const filterTitle = require('../utils/filter-title');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
||||||
const indexApi = new manticore.IndexApi(mantiClient);
|
|
||||||
|
|
||||||
const update = args.update;
|
|
||||||
|
|
||||||
async function fetchScenes() {
|
|
||||||
const scenes = await knex.raw(`
|
|
||||||
SELECT
|
|
||||||
releases.id AS id,
|
|
||||||
releases.title,
|
|
||||||
releases.created_at,
|
|
||||||
releases.date,
|
|
||||||
releases.entry_id,
|
|
||||||
releases.shoot_id,
|
|
||||||
scenes_meta.stashed,
|
|
||||||
entities.id as channel_id,
|
|
||||||
entities.slug as channel_slug,
|
|
||||||
entities.name as channel_name,
|
|
||||||
entities.alias as channel_aliases,
|
|
||||||
parents.id as network_id,
|
|
||||||
parents.slug as network_slug,
|
|
||||||
parents.name as network_name,
|
|
||||||
parents.alias as network_aliases,
|
|
||||||
studios.id as studio_id,
|
|
||||||
studios.slug as studio_slug,
|
|
||||||
studios.name as studio_name,
|
|
||||||
grandparents.id as parent_network_id,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
|
||||||
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
|
|
||||||
studios.showcased IS NOT false
|
|
||||||
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
|
|
||||||
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
|
|
||||||
AND (releases_summaries.batch_showcased IS NOT false)
|
|
||||||
AS showcased,
|
|
||||||
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
|
|
||||||
FROM releases
|
|
||||||
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
|
|
||||||
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
|
|
||||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
|
||||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
|
||||||
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
|
|
||||||
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
|
|
||||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
|
||||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
|
||||||
LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id
|
|
||||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
|
||||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
|
||||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
|
||||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
|
||||||
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
|
|
||||||
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
|
|
||||||
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
|
|
||||||
LEFT JOIN series ON series.id = series_scenes.serie_id
|
|
||||||
GROUP BY
|
|
||||||
releases.id,
|
|
||||||
releases.title,
|
|
||||||
releases.created_at,
|
|
||||||
releases.date,
|
|
||||||
releases.entry_id,
|
|
||||||
releases.shoot_id,
|
|
||||||
scenes_meta.stashed,
|
|
||||||
releases_summaries.batch_showcased,
|
|
||||||
entities.id,
|
|
||||||
entities.name,
|
|
||||||
entities.slug,
|
|
||||||
entities.alias,
|
|
||||||
parents.id,
|
|
||||||
parents.name,
|
|
||||||
parents.slug,
|
|
||||||
parents.alias,
|
|
||||||
grandparents.id,
|
|
||||||
studios.id,
|
|
||||||
studios.name,
|
|
||||||
studios.slug,
|
|
||||||
entities.showcased,
|
|
||||||
parents.showcased,
|
|
||||||
studios.showcased;
|
|
||||||
`);
|
|
||||||
|
|
||||||
return scenes.rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
if (update) {
|
|
||||||
await utilsApi.sql('drop table if exists scenes');
|
|
||||||
await utilsApi.sql(`create table scenes (
|
|
||||||
id int,
|
|
||||||
title text,
|
|
||||||
title_filtered text,
|
|
||||||
entry_id text,
|
|
||||||
shoot_id text,
|
|
||||||
channel_id int,
|
|
||||||
channel_name text,
|
|
||||||
channel_slug text,
|
|
||||||
network_id int,
|
|
||||||
network_name text,
|
|
||||||
network_slug text,
|
|
||||||
studio_id int,
|
|
||||||
studio_name text,
|
|
||||||
studio_slug text,
|
|
||||||
entity_ids multi,
|
|
||||||
actor_ids multi,
|
|
||||||
actors text,
|
|
||||||
tag_ids multi,
|
|
||||||
tags text,
|
|
||||||
movie_ids multi,
|
|
||||||
movies text,
|
|
||||||
serie_ids multi,
|
|
||||||
series text,
|
|
||||||
meta text,
|
|
||||||
date timestamp,
|
|
||||||
fingerprints text,
|
|
||||||
is_showcased bool,
|
|
||||||
created_at timestamp,
|
|
||||||
effective_date timestamp,
|
|
||||||
stashed int,
|
|
||||||
dupe_index int
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await utilsApi.sql('drop table if exists scenes_tags');
|
|
||||||
await utilsApi.sql(`create table scenes_tags (
|
|
||||||
id int,
|
|
||||||
scene_id int,
|
|
||||||
tag_id int,
|
|
||||||
actor_id int
|
|
||||||
)`);
|
|
||||||
|
|
||||||
console.log('Recreated scenes table');
|
|
||||||
console.log('Fetching scenes from primary database');
|
|
||||||
|
|
||||||
const scenes = await fetchScenes();
|
|
||||||
|
|
||||||
console.log('Fetched scenes from primary database');
|
|
||||||
|
|
||||||
const docs = scenes.flatMap((scene) => {
|
|
||||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
|
||||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
|
||||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
|
||||||
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
replace: {
|
|
||||||
index: 'scenes',
|
|
||||||
id: scene.id,
|
|
||||||
doc: {
|
|
||||||
title: scene.title || undefined,
|
|
||||||
title_filtered: filteredTitle || undefined,
|
|
||||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
|
||||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
|
||||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
|
||||||
is_showcased: scene.showcased,
|
|
||||||
entry_id: scene.entry_id || undefined,
|
|
||||||
shoot_id: scene.shoot_id || undefined,
|
|
||||||
channel_id: scene.channel_id,
|
|
||||||
channel_slug: scene.channel_slug,
|
|
||||||
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
|
|
||||||
network_id: scene.network_id || undefined,
|
|
||||||
network_slug: scene.network_slug || undefined,
|
|
||||||
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
|
|
||||||
studio_id: scene.studio_id || undefined,
|
|
||||||
studio_slug: scene.studio_slug || undefined,
|
|
||||||
studio_name: scene.studio_name || undefined,
|
|
||||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
|
||||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
|
||||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
|
||||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
|
||||||
tags: flatTags.join(' '),
|
|
||||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
|
||||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
|
||||||
serie_ids: scene.series.map((serie) => serie.f1),
|
|
||||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
|
||||||
fingerprints: scene.fingerprints.join(' '),
|
|
||||||
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
|
||||||
stashed: scene.stashed || 0,
|
|
||||||
dupe_index: scene.dupe_index || 0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
...scene.tags.map((tag) => ({
|
|
||||||
replace: {
|
|
||||||
index: 'scenes_tags',
|
|
||||||
// id: scene.id,
|
|
||||||
doc: {
|
|
||||||
scene_id: scene.id,
|
|
||||||
tag_id: tag.f1,
|
|
||||||
actor_id: tag.f5,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})),
|
|
||||||
];
|
|
||||||
});
|
|
||||||
|
|
||||||
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
|
|
||||||
chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
|
|
||||||
const acc = await chain;
|
|
||||||
const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
||||||
|
|
||||||
console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
|
|
||||||
|
|
||||||
return acc.concat(data.items);
|
|
||||||
}, Promise.resolve([]));
|
|
||||||
|
|
||||||
// console.log('data', accData);
|
|
||||||
}
|
|
||||||
|
|
||||||
knex.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
|
|
||||||
const knex = require('../knex');
|
|
||||||
const chunk = require('../utils/chunk');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
||||||
const indexApi = new manticore.IndexApi(mantiClient);
|
|
||||||
|
|
||||||
async function syncStashes(domain = 'scene') {
|
|
||||||
await utilsApi.sql(`truncate table ${domain}s_stashed`);
|
|
||||||
|
|
||||||
const stashes = await knex(`stashes_${domain}s`)
|
|
||||||
.select(
|
|
||||||
`stashes_${domain}s.id as stashed_id`,
|
|
||||||
`stashes_${domain}s.${domain}_id`,
|
|
||||||
'stashes.id as stash_id',
|
|
||||||
'stashes.user_id as user_id',
|
|
||||||
`stashes_${domain}s.created_at as created_at`,
|
|
||||||
)
|
|
||||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
|
||||||
|
|
||||||
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
|
|
||||||
await chain;
|
|
||||||
|
|
||||||
const stashDocs = stashChunk.map((stash) => ({
|
|
||||||
replace: {
|
|
||||||
index: `${domain}s_stashed`,
|
|
||||||
id: stash.stashed_id,
|
|
||||||
doc: {
|
|
||||||
[`${domain}_id`]: stash[`${domain}_id`],
|
|
||||||
stash_id: stash.stash_id,
|
|
||||||
user_id: stash.user_id,
|
|
||||||
created_at: Math.round(stash.created_at.getTime() / 1000),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
|
||||||
|
|
||||||
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
|
|
||||||
}, Promise.resolve());
|
|
||||||
}
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
await utilsApi.sql('drop table if exists scenes_stashed');
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
|
||||||
scene_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await utilsApi.sql('drop table if exists movies_stashed');
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
|
||||||
movie_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await utilsApi.sql('drop table if exists actors_stashed');
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
|
||||||
actor_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await syncStashes('scene');
|
|
||||||
await syncStashes('actor');
|
|
||||||
await syncStashes('movie');
|
|
||||||
|
|
||||||
console.log('Done!');
|
|
||||||
|
|
||||||
knex.destroy();
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
Reference in New Issue
Block a user