DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Removed Manticore sync tools, use traxxx-web repo.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-06-08 03:51:57 +02:00
parent 48f8c8da66
commit da7c9d4881
5 changed files with 2 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
seeds/00_tags.js
View File

@@ -647,6 +647,7 @@ const tags = [
{
name: 'MFF threesome',
slug: 'mff',
implies: ['threesome'],
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
group: 'group',
},
@@ -830,6 +831,7 @@ const tags = [
{
name: 'MFM threesome',
slug: 'mfm',
implies: ['threesome'],
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
group: 'group',
},

109
src/tools/manticore-actors.js
View File

@@ -1,109 +0,0 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const args = require('yargs').argv;
const knex = require('../knex');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const searchApi = new manticore.SearchApi(mantiClient);
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
const update = args.update;
async function fetchActors() {
// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
const actors = await knex.raw(`
SELECT
actors.*,
actors_meta.*,
date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
FROM actors
LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id
`);
return actors.rows;
}
async function init() {
if (update) {
await utilsApi.sql('drop table if exists actors');
await utilsApi.sql(`create table actors(
id int,
name text,
slug string,
entity_id int,
gender string,
date_of_birth timestamp,
country string,
has_avatar bool,
mass int,
height int,
cup string,
natural_boobs int,
penis_length int,
penis_girth int,
stashed int,
scenes int
) min_prefix_len = '3'`);
const actors = await fetchActors();
const docs = actors.map((actor) => ({
insert: {
index: 'actors',
id: actor.id,
doc: {
entity_id: actor.entity_id,
name: actor.name,
slug: actor.slug,
gender: actor.gender || undefined,
date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
has_avatar: !!actor.avatar_media_id,
country: actor.birth_country_alpha2 || undefined,
height: actor.height || undefined,
mass: actor.weight || undefined, // weight is a reserved keyword in manticore
cup: actor.cup || undefined,
natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
penis_length: actor.penis_length || undefined,
penis_girth: actor.penis_girth || undefined,
stashed: actor.stashed || 0,
scenes: actor.scenes || 0,
},
},
}));
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
console.log(error);
});
console.log('data', data);
knex.destroy();
return;
}
const result = await searchApi.search({
index: 'actors',
query: {
equals: {
has_avatar: 1,
},
},
limit: 3,
sort: [{ slug: 'asc' }],
});
console.log(result);
console.log(result.hits?.hits);
knex.destroy();
}
init();

165
src/tools/manticore-movies.js
View File

@@ -1,165 +0,0 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const args = require('yargs').argv;
const { format } = require('date-fns');
const knex = require('../knex');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
// const searchApi = new manticore.SearchApi(mantiClient);
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
const update = args.update;
async function fetchMovies() {
const movies = await knex.raw(`
SELECT
movies.id AS id,
movies.title,
movies.created_at,
movies.date,
movies_meta.stashed,
entities.id as channel_id,
entities.slug as channel_slug,
entities.name as channel_name,
parents.id as network_id,
parents.slug as network_slug,
parents.name as network_name,
movies_covers IS NOT NULL as has_cover,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
FROM movies
LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
LEFT JOIN entities ON movies.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
GROUP BY
movies.id,
movies.title,
movies.created_at,
movies.date,
movies_meta.stashed,
movies_meta.stashed_scenes,
movies_meta.stashed_total,
entities.id,
entities.name,
entities.slug,
entities.alias,
parents.id,
parents.name,
parents.slug,
parents.alias,
movies_covers.*
`);
return movies.rows;
}
async function init() {
if (update) {
await utilsApi.sql('drop table if exists movies');
await utilsApi.sql(`create table movies (
id int,
title text,
title_filtered text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
meta text,
date timestamp,
has_cover bool,
created_at timestamp,
effective_date timestamp,
stashed int,
stashed_scenes int,
stashed_total int,
dupe_index int
)`);
const movies = await fetchMovies();
console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index));
const docs = movies.map((movie) => {
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
id: tag.f1,
name: tag.f2,
priority: tag.f3,
alias: tag.f4,
}])));
const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
return {
replace: {
index: 'movies',
id: movie.id,
doc: {
title: movie.title || undefined,
title_filtered: filteredTitle || undefined,
date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
created_at: Math.round(movie.created_at.getTime() / 1000),
effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
channel_id: movie.channel_id,
channel_slug: movie.channel_slug,
channel_name: movie.channel_name,
network_id: movie.network_id || undefined,
network_slug: movie.network_slug || undefined,
network_name: movie.network_name || undefined,
entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: movie.actors.map((actor) => actor.f1),
actors: movie.actors.map((actor) => actor.f2).join(),
tag_ids: combinedTags.map((tag) => tag.id),
tags: flatTags.join(' '),
has_cover: movie.has_cover,
meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: movie.stashed || 0,
stashed_scenes: movie.stashed_scenes || 0,
stashed_total: movie.stashed_total || 0,
dupe_index: movie.dupe_index || 0,
},
},
};
});
console.log(docs.map((doc) => doc.replace));
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log('data', data);
}
knex.destroy();
}
init();

228
src/tools/manticore-scenes.js
View File

@@ -1,228 +0,0 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const args = require('yargs').argv;
const { format } = require('date-fns');
const knex = require('../knex');
const chunk = require('../utils/chunk');
const filterTitle = require('../utils/filter-title');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
const update = args.update;
async function fetchScenes() {
const scenes = await knex.raw(`
SELECT
releases.id AS id,
releases.title,
releases.created_at,
releases.date,
releases.entry_id,
releases.shoot_id,
scenes_meta.stashed,
entities.id as channel_id,
entities.slug as channel_slug,
entities.name as channel_name,
entities.alias as channel_aliases,
parents.id as network_id,
parents.slug as network_slug,
parents.name as network_name,
parents.alias as network_aliases,
studios.id as studio_id,
studios.slug as studio_slug,
studios.name as studio_name,
grandparents.id as parent_network_id,
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
studios.showcased IS NOT false
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
AND (releases_summaries.batch_showcased IS NOT false)
AS showcased,
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
FROM releases
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
LEFT JOIN entities ON releases.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
LEFT JOIN series ON series.id = series_scenes.serie_id
GROUP BY
releases.id,
releases.title,
releases.created_at,
releases.date,
releases.entry_id,
releases.shoot_id,
scenes_meta.stashed,
releases_summaries.batch_showcased,
entities.id,
entities.name,
entities.slug,
entities.alias,
parents.id,
parents.name,
parents.slug,
parents.alias,
grandparents.id,
studios.id,
studios.name,
studios.slug,
entities.showcased,
parents.showcased,
studios.showcased;
`);
return scenes.rows;
}
async function init() {
if (update) {
await utilsApi.sql('drop table if exists scenes');
await utilsApi.sql(`create table scenes (
id int,
title text,
title_filtered text,
entry_id text,
shoot_id text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
studio_id int,
studio_name text,
studio_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
movie_ids multi,
movies text,
serie_ids multi,
series text,
meta text,
date timestamp,
fingerprints text,
is_showcased bool,
created_at timestamp,
effective_date timestamp,
stashed int,
dupe_index int
)`);
await utilsApi.sql('drop table if exists scenes_tags');
await utilsApi.sql(`create table scenes_tags (
id int,
scene_id int,
tag_id int,
actor_id int
)`);
console.log('Recreated scenes table');
console.log('Fetching scenes from primary database');
const scenes = await fetchScenes();
console.log('Fetched scenes from primary database');
const docs = scenes.flatMap((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
return [
{
replace: {
index: 'scenes',
id: scene.id,
doc: {
title: scene.title || undefined,
title_filtered: filteredTitle || undefined,
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
created_at: Math.round(scene.created_at.getTime() / 1000),
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
is_showcased: scene.showcased,
entry_id: scene.entry_id || undefined,
shoot_id: scene.shoot_id || undefined,
channel_id: scene.channel_id,
channel_slug: scene.channel_slug,
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
network_id: scene.network_id || undefined,
network_slug: scene.network_slug || undefined,
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
studio_id: scene.studio_id || undefined,
studio_slug: scene.studio_slug || undefined,
studio_name: scene.studio_name || undefined,
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
actor_ids: scene.actors.map((actor) => actor.f1),
actors: scene.actors.map((actor) => actor.f2).join(),
tag_ids: scene.tags.map((tag) => tag.f1),
tags: flatTags.join(' '),
movie_ids: scene.movies.map((movie) => movie.f1),
movies: scene.movies.map((movie) => movie.f2).join(' '),
serie_ids: scene.series.map((serie) => serie.f1),
series: scene.series.map((serie) => serie.f2).join(' '),
fingerprints: scene.fingerprints.join(' '),
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
stashed: scene.stashed || 0,
dupe_index: scene.dupe_index || 0,
},
},
},
...scene.tags.map((tag) => ({
replace: {
index: 'scenes_tags',
// id: scene.id,
doc: {
scene_id: scene.id,
tag_id: tag.f1,
actor_id: tag.f5,
},
},
})),
];
});
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
const acc = await chain;
const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
return acc.concat(data.items);
}, Promise.resolve([]));
// console.log('data', accData);
}
knex.destroy();
}
init();

88
src/tools/manticore-stashes.js
View File

@@ -1,88 +0,0 @@
'use strict';
const config = require('config');
const manticore = require('manticoresearch');
const knex = require('../knex');
const chunk = require('../utils/chunk');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const utilsApi = new manticore.UtilsApi(mantiClient);
const indexApi = new manticore.IndexApi(mantiClient);
async function syncStashes(domain = 'scene') {
await utilsApi.sql(`truncate table ${domain}s_stashed`);
const stashes = await knex(`stashes_${domain}s`)
.select(
`stashes_${domain}s.id as stashed_id`,
`stashes_${domain}s.${domain}_id`,
'stashes.id as stash_id',
'stashes.user_id as user_id',
`stashes_${domain}s.created_at as created_at`,
)
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
await chain;
const stashDocs = stashChunk.map((stash) => ({
replace: {
index: `${domain}s_stashed`,
id: stash.stashed_id,
doc: {
[`${domain}_id`]: stash[`${domain}_id`],
stash_id: stash.stash_id,
user_id: stash.user_id,
created_at: Math.round(stash.created_at.getTime() / 1000),
},
},
}));
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
}, Promise.resolve());
}
async function init() {
await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql(`create table if not exists scenes_stashed (
scene_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists movies_stashed');
await utilsApi.sql(`create table if not exists movies_stashed (
movie_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql('drop table if exists actors_stashed');
await utilsApi.sql(`create table if not exists actors_stashed (
actor_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await syncStashes('scene');
await syncStashes('actor');
await syncStashes('movie');
console.log('Done!');
knex.destroy();
}
init();