DebaucheryLibrarian/traxxx-web
1
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Added manticore sync all tool, updated manticore driver.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-11-12 02:18:13 +01:00
parent de05075def
commit f5e3a8a2ae
7 changed files with 175 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+3 -3
View File
@@ -14167,9 +14167,9 @@
} }
}, },
"node_modules/manticoresearch": { "node_modules/manticoresearch": {
"version": "10.0.0", "version": "10.2.0",
"resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.0.0.tgz", "resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.2.0.tgz",
"integrity": "sha512-a6eiAImHjhLLJgDYFRJfw2k4UL+y7ucchCbNN7q3t+6Fo4TvYf+vbS3HOcRv2h5wIFo6yaZ5N09ztKOOGqlhNw==", "integrity": "sha512-dzXZwrSVizGjV4yYjEsuzq4TQqqh9FDaCu173p7wxq2xpHwiSp5FTxiBJcMTUQa1K9Di41RUoDTMWAjUjYYjFA==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@babel/cli": "^7.0.0", "@babel/cli": "^7.0.0",
tools/manticore-actors.js
+33 -40
View File
@@ -1,49 +1,42 @@
import argv from '../src/argv.js'; import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js'; import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreActors } from '../src/sync.js'; import { syncManticoreActors } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() { export async function rebuildActors() {
if (argv.update) { await utilsApi.sql('drop table if exists actors');
await utilsApi.sql('drop table if exists actors'); // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s await utilsApi.sql(`create table actors(
await utilsApi.sql(`create table actors( id int,
id int, name text,
name text, aliases text,
aliases text, slug string,
slug string, entity_id int,
entity_id int, alias_for int,
alias_for int, gender string,
gender string, date_of_birth bigint,
date_of_birth bigint, country string,
country string, has_avatar bool,
has_avatar bool, mass int,
mass int, height int,
height int, cup string,
cup string, natural_boobs int,
natural_boobs int, penis_length int,
penis_length int, penis_girth int,
penis_girth int, stashed int,
stashed int, scenes int
scenes int ) min_prefix_len = '3'`);
) min_prefix_len = '3'`);
console.log('Recreated actors table, syncing actors...'); console.log('Recreated actors table, syncing actors...');
const data = await syncManticoreActors(); await syncManticoreActors();
console.log('data', data);
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
return;
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
} }
init(); if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildActors();
}
});
}
tools/manticore-movies.js
+38 -40
View File
@@ -1,48 +1,46 @@
import argv from '../src/argv.js'; import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js'; import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreMovies } from '../src/sync.js'; import { syncManticoreMovies } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() { export async function rebuildMovies() {
if (argv.update) { await utilsApi.sql('drop table if exists movies');
await utilsApi.sql('drop table if exists movies'); // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s await utilsApi.sql(`create table movies (
await utilsApi.sql(`create table movies ( id int,
id int, title text,
title text, title_filtered text,
title_filtered text, channel_id int,
channel_id int, channel_name text,
channel_name text, channel_slug text,
channel_slug text, network_id int,
network_id int, network_name text,
network_name text, network_slug text,
network_slug text, entity_ids multi,
entity_ids multi, actor_ids multi,
actor_ids multi, actors text,
actors text, tag_ids multi,
tag_ids multi, tags text,
tags text, meta text,
meta text, date bigint,
date bigint, has_cover bool,
has_cover bool, created_at bigint,
created_at bigint, effective_date bigint,
effective_date bigint, stashed int,
stashed int, stashed_scenes int,
stashed_scenes int, stashed_total int,
stashed_total int, dupe_index int
dupe_index int )`);
)`);
console.log('Recreated movies tables, syncing movies...'); console.log('Recreated movies tables, syncing movies...');
const data = await syncManticoreMovies(); await syncManticoreMovies();
console.log('data', data);
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
} }
init(); if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildMovies();
}
});
}
tools/manticore-scenes.js
+61 -61
View File
@@ -1,71 +1,71 @@
import argv from '../src/argv.js'; import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js'; import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreScenes } from '../src/sync.js'; import { syncManticoreScenes } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() { export async function rebuildScenes() {
if (argv.update) { await utilsApi.sql('drop table if exists scenes');
await utilsApi.sql('drop table if exists scenes'); // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s await utilsApi.sql(`create table scenes (
await utilsApi.sql(`create table scenes ( id int,
id int, title text,
title text, title_filtered text,
title_filtered text, entry_id text,
entry_id text, shoot_id text,
shoot_id text, channel_id int,
channel_id int, channel_name text,
channel_name text, channel_slug text,
channel_slug text, network_id int,
network_id int, network_name text,
network_name text, network_slug text,
network_slug text, studio_id int,
studio_id int, studio_name text,
studio_name text, studio_slug text,
studio_slug text, entity_ids multi,
entity_ids multi, actor_ids multi,
actor_ids multi, actors text,
actors text, actor_count int,
actor_count int, female_count int,
female_count int, male_count int,
male_count int, trans_count int,
trans_count int, unidentified_count int,
unidentified_count int, tag_ids multi,
tag_ids multi, assigned_tag_ids multi64,
assigned_tag_ids multi64, tags text,
tags text, movie_ids multi,
movie_ids multi, movies text,
movies text, serie_ids multi,
serie_ids multi, series text,
series text, meta text,
meta text, date bigint,
date bigint, fingerprints text,
fingerprints text, is_showcased bool,
is_showcased bool, created_at bigint,
created_at bigint, effective_date bigint,
effective_date bigint, stashed int,
stashed int, dupe_index int
dupe_index int )`);
)`);
await utilsApi.sql('drop table if exists scenes_tags'); await utilsApi.sql('drop table if exists scenes_tags');
/* legacy, using packed decimal keys now /* legacy, using packed decimal keys now
await utilsApi.sql(`create table scenes_tags ( await utilsApi.sql(`create table scenes_tags (
id int, id int,
scene_id int, scene_id int,
tag_id int, tag_id int,
actor_id int actor_id int
)`); )`);
*/ */
console.log('Recreated scenes tables, syncing scenes...'); console.log('Recreated scenes tables, syncing scenes...');
await syncManticoreScenes(); await syncManticoreScenes();
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
} }
init(); if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildScenes();
}
});
}
tools/manticore-stashes.js
+5 -7
View File
@@ -1,9 +1,8 @@
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js'; import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncStashes } from '../src/sync.js'; import { syncStashes } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() { export async function rebuildStashes() {
await utilsApi.sql('drop table if exists scenes_stashed'); await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql(`create table if not exists scenes_stashed ( await utilsApi.sql(`create table if not exists scenes_stashed (
@@ -36,9 +35,8 @@ async function init() {
await syncStashes('scene'); await syncStashes('scene');
await syncStashes('actor'); await syncStashes('actor');
await syncStashes('movie'); await syncStashes('movie');
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
} }
init(); if (import.meta.main) {
runTool(rebuildStashes);
}
tools/run-tool.js
+19
View File
@@ -0,0 +1,19 @@
import process from 'node:process';
import { knexOwner as knex } from '../src/knex.js';
import redis from '../src/redis.js';
// runs a command line tool and closes the connections afterwards, the knex pool and redis would otherwise keep the process alive
export default async function runTool(tool) {
try {
await tool();
await knex.destroy();
await redis.quit();
}
catch (error) {
// the manticore client rejects with a plain response object rather than an Error, print the server's message
console.error(error instanceof Error ? error : error?.body?.error || error?.body || error);
process.exit(1);
}
}
tools/sync-all.js
+16
View File
@@ -0,0 +1,16 @@
import { rebuildActors } from './manticore-actors.js';
import { rebuildMovies } from './manticore-movies.js';
import { rebuildScenes } from './manticore-scenes.js';
import { rebuildStashes } from './manticore-stashes.js';
import runTool from './run-tool.js';
// recreates and fills every manticore table from postgres, i.e. after a manticore upgrade or on a fresh data directory
runTool(async () => {
// stashes first, the scenes sync also writes scene stashes and fails if their table doesn't exist yet
await rebuildStashes();
await rebuildScenes();
await rebuildMovies();
await rebuildActors();
console.log('Rebuilt all manticore tables');
});