diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 21af5ea3..476edf67 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -647,6 +647,7 @@ const tags = [ { name: 'MFF threesome', slug: 'mff', + implies: ['threesome'], description: 'A threesome with two women and one guy, in which the women have sex with eachother.', group: 'group', }, @@ -830,6 +831,7 @@ const tags = [ { name: 'MFM threesome', slug: 'mfm', + implies: ['threesome'], description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.', group: 'group', }, diff --git a/src/tools/manticore-actors.js b/src/tools/manticore-actors.js deleted file mode 100644 index 13bfb2df..00000000 --- a/src/tools/manticore-actors.js +++ /dev/null @@ -1,109 +0,0 @@ -'use strict'; - -const config = require('config'); -const manticore = require('manticoresearch'); -const args = require('yargs').argv; - -const knex = require('../knex'); - -const mantiClient = new manticore.ApiClient(); - -mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`; - -const searchApi = new manticore.SearchApi(mantiClient); - -const utilsApi = new manticore.UtilsApi(mantiClient); -const indexApi = new manticore.IndexApi(mantiClient); - -const update = args.update; - -async function fetchActors() { - // manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC... - const actors = await knex.raw(` - SELECT - actors.*, - actors_meta.*, - date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob - FROM actors - LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id - `); - - return actors.rows; -} - -async function init() { - if (update) { - await utilsApi.sql('drop table if exists actors'); - await utilsApi.sql(`create table actors( - id int, - name text, - slug string, - entity_id int, - gender string, - date_of_birth timestamp, - country string, - has_avatar bool, - mass int, - height int, - cup string, - natural_boobs int, - penis_length int, - penis_girth int, - stashed int, - scenes int - ) min_prefix_len = '3'`); - - const actors = await fetchActors(); - - const docs = actors.map((actor) => ({ - insert: { - index: 'actors', - id: actor.id, - doc: { - entity_id: actor.entity_id, - name: actor.name, - slug: actor.slug, - gender: actor.gender || undefined, - date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined, - has_avatar: !!actor.avatar_media_id, - country: actor.birth_country_alpha2 || undefined, - height: actor.height || undefined, - mass: actor.weight || undefined, // weight is a reserved keyword in manticore - cup: actor.cup || undefined, - natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown - penis_length: actor.penis_length || undefined, - penis_girth: actor.penis_girth || undefined, - stashed: actor.stashed || 0, - scenes: actor.scenes || 0, - }, - }, - })); - - const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => { - console.log(error); - }); - - console.log('data', data); - knex.destroy(); - - return; - } - - const result = await searchApi.search({ - index: 'actors', - query: { - equals: { - has_avatar: 1, - }, - }, - limit: 3, - sort: [{ slug: 'asc' }], - }); - - console.log(result); - console.log(result.hits?.hits); - - knex.destroy(); -} - -init(); diff --git a/src/tools/manticore-movies.js b/src/tools/manticore-movies.js deleted file mode 100644 index a8a4bc31..00000000 --- a/src/tools/manticore-movies.js +++ /dev/null @@ -1,165 +0,0 @@ -'use strict'; - -const config = require('config'); -const manticore = require('manticoresearch'); -const args = require('yargs').argv; -const { format } = require('date-fns'); - -const knex = require('../knex'); - -const mantiClient = new manticore.ApiClient(); - -mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`; - -// const searchApi = new manticore.SearchApi(mantiClient); - -const utilsApi = new manticore.UtilsApi(mantiClient); -const indexApi = new manticore.IndexApi(mantiClient); - -const update = args.update; - -async function fetchMovies() { - const movies = await knex.raw(` - SELECT - movies.id AS id, - movies.title, - movies.created_at, - movies.date, - movies_meta.stashed, - entities.id as channel_id, - entities.slug as channel_slug, - entities.name as channel_name, - parents.id as network_id, - parents.slug as network_slug, - parents.name as network_name, - movies_covers IS NOT NULL as has_cover, - COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, - COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, - COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags, - row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index - FROM movies - LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id - LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id - LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id - LEFT JOIN entities ON movies.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id - LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true - LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id - GROUP BY - movies.id, - movies.title, - movies.created_at, - movies.date, - movies_meta.stashed, - movies_meta.stashed_scenes, - movies_meta.stashed_total, - entities.id, - entities.name, - entities.slug, - entities.alias, - parents.id, - parents.name, - parents.slug, - parents.alias, - movies_covers.* - `); - - return movies.rows; -} - -async function init() { - if (update) { - await utilsApi.sql('drop table if exists movies'); - await utilsApi.sql(`create table movies ( - id int, - title text, - title_filtered text, - channel_id int, - channel_name text, - channel_slug text, - network_id int, - network_name text, - network_slug text, - entity_ids multi, - actor_ids multi, - actors text, - tag_ids multi, - tags text, - meta text, - date timestamp, - has_cover bool, - created_at timestamp, - effective_date timestamp, - stashed int, - stashed_scenes int, - stashed_total int, - dupe_index int - )`); - - const movies = await fetchMovies(); - - console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index)); - - const docs = movies.map((movie) => { - const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, { - id: tag.f1, - name: tag.f2, - priority: tag.f3, - alias: tag.f4, - }]))); - - const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc. - const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results - const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' '); - - return { - replace: { - index: 'movies', - id: movie.id, - doc: { - title: movie.title || undefined, - title_filtered: filteredTitle || undefined, - date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined, - created_at: Math.round(movie.created_at.getTime() / 1000), - effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000), - channel_id: movie.channel_id, - channel_slug: movie.channel_slug, - channel_name: movie.channel_name, - network_id: movie.network_id || undefined, - network_slug: movie.network_slug || undefined, - network_name: movie.network_name || undefined, - entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN - actor_ids: movie.actors.map((actor) => actor.f1), - actors: movie.actors.map((actor) => actor.f2).join(), - tag_ids: combinedTags.map((tag) => tag.id), - tags: flatTags.join(' '), - has_cover: movie.has_cover, - meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined, - stashed: movie.stashed || 0, - stashed_scenes: movie.stashed_scenes || 0, - stashed_total: movie.stashed_total || 0, - dupe_index: movie.dupe_index || 0, - }, - }, - }; - }); - - console.log(docs.map((doc) => doc.replace)); - - const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); - - console.log('data', data); - } - - knex.destroy(); -} - -init(); diff --git a/src/tools/manticore-scenes.js b/src/tools/manticore-scenes.js deleted file mode 100644 index 981c01e0..00000000 --- a/src/tools/manticore-scenes.js +++ /dev/null @@ -1,228 +0,0 @@ -'use strict'; - -const config = require('config'); -const manticore = require('manticoresearch'); -const args = require('yargs').argv; -const { format } = require('date-fns'); - -const knex = require('../knex'); -const chunk = require('../utils/chunk'); -const filterTitle = require('../utils/filter-title'); - -const mantiClient = new manticore.ApiClient(); - -mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`; - -const utilsApi = new manticore.UtilsApi(mantiClient); -const indexApi = new manticore.IndexApi(mantiClient); - -const update = args.update; - -async function fetchScenes() { - const scenes = await knex.raw(` - SELECT - releases.id AS id, - releases.title, - releases.created_at, - releases.date, - releases.entry_id, - releases.shoot_id, - scenes_meta.stashed, - entities.id as channel_id, - entities.slug as channel_slug, - entities.name as channel_name, - entities.alias as channel_aliases, - parents.id as network_id, - parents.slug as network_slug, - parents.name as network_name, - parents.alias as network_aliases, - studios.id as studio_id, - studios.slug as studio_slug, - studios.name as studio_name, - grandparents.id as parent_network_id, - COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, - COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags, - COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies, - COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series, - COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints, - studios.showcased IS NOT false - AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true) - AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true) - AND (releases_summaries.batch_showcased IS NOT false) - AS showcased, - row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index - FROM releases - LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id - LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id - LEFT JOIN entities ON releases.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id - LEFT JOIN entities AS studios ON studios.id = releases.studio_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id - LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id - LEFT JOIN movies ON movies.id = movies_scenes.movie_id - LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id - LEFT JOIN series ON series.id = series_scenes.serie_id - GROUP BY - releases.id, - releases.title, - releases.created_at, - releases.date, - releases.entry_id, - releases.shoot_id, - scenes_meta.stashed, - releases_summaries.batch_showcased, - entities.id, - entities.name, - entities.slug, - entities.alias, - parents.id, - parents.name, - parents.slug, - parents.alias, - grandparents.id, - studios.id, - studios.name, - studios.slug, - entities.showcased, - parents.showcased, - studios.showcased; - `); - - return scenes.rows; -} - -async function init() { - if (update) { - await utilsApi.sql('drop table if exists scenes'); - await utilsApi.sql(`create table scenes ( - id int, - title text, - title_filtered text, - entry_id text, - shoot_id text, - channel_id int, - channel_name text, - channel_slug text, - network_id int, - network_name text, - network_slug text, - studio_id int, - studio_name text, - studio_slug text, - entity_ids multi, - actor_ids multi, - actors text, - tag_ids multi, - tags text, - movie_ids multi, - movies text, - serie_ids multi, - series text, - meta text, - date timestamp, - fingerprints text, - is_showcased bool, - created_at timestamp, - effective_date timestamp, - stashed int, - dupe_index int - )`); - - await utilsApi.sql('drop table if exists scenes_tags'); - await utilsApi.sql(`create table scenes_tags ( - id int, - scene_id int, - tag_id int, - actor_id int - )`); - - console.log('Recreated scenes table'); - console.log('Fetching scenes from primary database'); - - const scenes = await fetchScenes(); - - console.log('Fetched scenes from primary database'); - - const docs = scenes.flatMap((scene) => { - const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc. - const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results - const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]); - - return [ - { - replace: { - index: 'scenes', - id: scene.id, - doc: { - title: scene.title || undefined, - title_filtered: filteredTitle || undefined, - date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined, - created_at: Math.round(scene.created_at.getTime() / 1000), - effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000), - is_showcased: scene.showcased, - entry_id: scene.entry_id || undefined, - shoot_id: scene.shoot_id || undefined, - channel_id: scene.channel_id, - channel_slug: scene.channel_slug, - channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '), - network_id: scene.network_id || undefined, - network_slug: scene.network_slug || undefined, - network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined, - studio_id: scene.studio_id || undefined, - studio_slug: scene.studio_slug || undefined, - studio_name: scene.studio_name || undefined, - entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN - actor_ids: scene.actors.map((actor) => actor.f1), - actors: scene.actors.map((actor) => actor.f2).join(), - tag_ids: scene.tags.map((tag) => tag.f1), - tags: flatTags.join(' '), - movie_ids: scene.movies.map((movie) => movie.f1), - movies: scene.movies.map((movie) => movie.f2).join(' '), - serie_ids: scene.series.map((serie) => serie.f1), - series: scene.series.map((serie) => serie.f2).join(' '), - fingerprints: scene.fingerprints.join(' '), - meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined, - stashed: scene.stashed || 0, - dupe_index: scene.dupe_index || 0, - }, - }, - }, - ...scene.tags.map((tag) => ({ - replace: { - index: 'scenes_tags', - // id: scene.id, - doc: { - scene_id: scene.id, - tag_id: tag.f1, - actor_id: tag.f5, - }, - }, - })), - ]; - }); - - // const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => { - chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => { - const acc = await chain; - const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n')); - - console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`); - - return acc.concat(data.items); - }, Promise.resolve([])); - - // console.log('data', accData); - } - - knex.destroy(); -} - -init(); diff --git a/src/tools/manticore-stashes.js b/src/tools/manticore-stashes.js deleted file mode 100644 index 98b9d6f9..00000000 --- a/src/tools/manticore-stashes.js +++ /dev/null @@ -1,88 +0,0 @@ -'use strict'; - -const config = require('config'); -const manticore = require('manticoresearch'); - -const knex = require('../knex'); -const chunk = require('../utils/chunk'); - -const mantiClient = new manticore.ApiClient(); - -mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`; - -const utilsApi = new manticore.UtilsApi(mantiClient); -const indexApi = new manticore.IndexApi(mantiClient); - -async function syncStashes(domain = 'scene') { - await utilsApi.sql(`truncate table ${domain}s_stashed`); - - const stashes = await knex(`stashes_${domain}s`) - .select( - `stashes_${domain}s.id as stashed_id`, - `stashes_${domain}s.${domain}_id`, - 'stashes.id as stash_id', - 'stashes.user_id as user_id', - `stashes_${domain}s.created_at as created_at`, - ) - .leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`); - - await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => { - await chain; - - const stashDocs = stashChunk.map((stash) => ({ - replace: { - index: `${domain}s_stashed`, - id: stash.stashed_id, - doc: { - [`${domain}_id`]: stash[`${domain}_id`], - stash_id: stash.stash_id, - user_id: stash.user_id, - created_at: Math.round(stash.created_at.getTime() / 1000), - }, - }, - })); - - await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n')); - - console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`); - }, Promise.resolve()); -} - -async function init() { - await utilsApi.sql('drop table if exists scenes_stashed'); - - await utilsApi.sql(`create table if not exists scenes_stashed ( - scene_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await utilsApi.sql('drop table if exists movies_stashed'); - - await utilsApi.sql(`create table if not exists movies_stashed ( - movie_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await utilsApi.sql('drop table if exists actors_stashed'); - - await utilsApi.sql(`create table if not exists actors_stashed ( - actor_id int, - stash_id int, - user_id int, - created_at timestamp - )`); - - await syncStashes('scene'); - await syncStashes('actor'); - await syncStashes('movie'); - - console.log('Done!'); - - knex.destroy(); -} - -init();