DebaucheryLibrarian/traxxx-web
1
1
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Added manticore sync all tool, updated manticore driver.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-11-12 02:18:13 +01:00
parent de05075def
commit f5e3a8a2ae
7 changed files with 175 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+3 -3
View File
@@ -14167,9 +14167,9 @@
}
},
"node_modules/manticoresearch": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.0.0.tgz",
"integrity": "sha512-a6eiAImHjhLLJgDYFRJfw2k4UL+y7ucchCbNN7q3t+6Fo4TvYf+vbS3HOcRv2h5wIFo6yaZ5N09ztKOOGqlhNw==",
"version": "10.2.0",
"resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.2.0.tgz",
"integrity": "sha512-dzXZwrSVizGjV4yYjEsuzq4TQqqh9FDaCu173p7wxq2xpHwiSp5FTxiBJcMTUQa1K9Di41RUoDTMWAjUjYYjFA==",
"license": "MIT",
"dependencies": {
"@babel/cli": "^7.0.0",
tools/manticore-actors.js
+33 -40
View File
@@ -1,49 +1,42 @@
import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreActors } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() {
if (argv.update) {
await utilsApi.sql('drop table if exists actors');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table actors(
id int,
name text,
aliases text,
slug string,
entity_id int,
alias_for int,
gender string,
date_of_birth bigint,
country string,
has_avatar bool,
mass int,
height int,
cup string,
natural_boobs int,
penis_length int,
penis_girth int,
stashed int,
scenes int
) min_prefix_len = '3'`);
export async function rebuildActors() {
await utilsApi.sql('drop table if exists actors');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table actors(
id int,
name text,
aliases text,
slug string,
entity_id int,
alias_for int,
gender string,
date_of_birth bigint,
country string,
has_avatar bool,
mass int,
height int,
cup string,
natural_boobs int,
penis_length int,
penis_girth int,
stashed int,
scenes int
) min_prefix_len = '3'`);
console.log('Recreated actors table, syncing actors...');
console.log('Recreated actors table, syncing actors...');
const data = await syncManticoreActors();
console.log('data', data);
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
return;
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
await syncManticoreActors();
}
init();
if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildActors();
}
});
}
tools/manticore-movies.js
+38 -40
View File
@@ -1,48 +1,46 @@
import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreMovies } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() {
if (argv.update) {
await utilsApi.sql('drop table if exists movies');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table movies (
id int,
title text,
title_filtered text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
meta text,
date bigint,
has_cover bool,
created_at bigint,
effective_date bigint,
stashed int,
stashed_scenes int,
stashed_total int,
dupe_index int
)`);
export async function rebuildMovies() {
await utilsApi.sql('drop table if exists movies');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table movies (
id int,
title text,
title_filtered text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
meta text,
date bigint,
has_cover bool,
created_at bigint,
effective_date bigint,
stashed int,
stashed_scenes int,
stashed_total int,
dupe_index int
)`);
console.log('Recreated movies tables, syncing movies...');
console.log('Recreated movies tables, syncing movies...');
const data = await syncManticoreMovies();
console.log('data', data);
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
await syncManticoreMovies();
}
init();
if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildMovies();
}
});
}
tools/manticore-scenes.js
+61 -61
View File
@@ -1,71 +1,71 @@
import argv from '../src/argv.js';
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncManticoreScenes } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() {
if (argv.update) {
await utilsApi.sql('drop table if exists scenes');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table scenes (
id int,
title text,
title_filtered text,
entry_id text,
shoot_id text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
studio_id int,
studio_name text,
studio_slug text,
entity_ids multi,
actor_ids multi,
actors text,
actor_count int,
female_count int,
male_count int,
trans_count int,
unidentified_count int,
tag_ids multi,
assigned_tag_ids multi64,
tags text,
movie_ids multi,
movies text,
serie_ids multi,
series text,
meta text,
date bigint,
fingerprints text,
is_showcased bool,
created_at bigint,
effective_date bigint,
stashed int,
dupe_index int
)`);
export async function rebuildScenes() {
await utilsApi.sql('drop table if exists scenes');
// bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s
await utilsApi.sql(`create table scenes (
id int,
title text,
title_filtered text,
entry_id text,
shoot_id text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
studio_id int,
studio_name text,
studio_slug text,
entity_ids multi,
actor_ids multi,
actors text,
actor_count int,
female_count int,
male_count int,
trans_count int,
unidentified_count int,
tag_ids multi,
assigned_tag_ids multi64,
tags text,
movie_ids multi,
movies text,
serie_ids multi,
series text,
meta text,
date bigint,
fingerprints text,
is_showcased bool,
created_at bigint,
effective_date bigint,
stashed int,
dupe_index int
)`);
await utilsApi.sql('drop table if exists scenes_tags');
await utilsApi.sql('drop table if exists scenes_tags');
/* legacy, using packed decimal keys now
await utilsApi.sql(`create table scenes_tags (
id int,
scene_id int,
tag_id int,
actor_id int
)`);
*/
/* legacy, using packed decimal keys now
await utilsApi.sql(`create table scenes_tags (
id int,
scene_id int,
tag_id int,
actor_id int
)`);
*/
console.log('Recreated scenes tables, syncing scenes...');
console.log('Recreated scenes tables, syncing scenes...');
await syncManticoreScenes();
}
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
await syncManticoreScenes();
}
init();
if (import.meta.main) {
runTool(async () => {
if (argv.update) {
await rebuildScenes();
}
});
}
tools/manticore-stashes.js
+5 -7
View File
@@ -1,9 +1,8 @@
import { knexOwner as knex } from '../src/knex.js';
import { utilsApi } from '../src/manticore.js';
import redis from '../src/redis.js';
import { syncStashes } from '../src/sync.js';
import runTool from './run-tool.js';
async function init() {
export async function rebuildStashes() {
await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql(`create table if not exists scenes_stashed (
@@ -36,9 +35,8 @@ async function init() {
await syncStashes('scene');
await syncStashes('actor');
await syncStashes('movie');
await knex.destroy();
await redis.quit(); // sync imports connect to redis, which keeps the process alive
}
init();
if (import.meta.main) {
runTool(rebuildStashes);
}
tools/run-tool.js
+19
View File
@@ -0,0 +1,19 @@
import process from 'node:process';
import { knexOwner as knex } from '../src/knex.js';
import redis from '../src/redis.js';
// runs a command line tool and closes the connections afterwards, the knex pool and redis would otherwise keep the process alive
export default async function runTool(tool) {
try {
await tool();
await knex.destroy();
await redis.quit();
}
catch (error) {
// the manticore client rejects with a plain response object rather than an Error, print the server's message
console.error(error instanceof Error ? error : error?.body?.error || error?.body || error);
process.exit(1);
}
}
tools/sync-all.js
+16
View File
@@ -0,0 +1,16 @@
import { rebuildActors } from './manticore-actors.js';
import { rebuildMovies } from './manticore-movies.js';
import { rebuildScenes } from './manticore-scenes.js';
import { rebuildStashes } from './manticore-stashes.js';
import runTool from './run-tool.js';
// recreates and fills every manticore table from postgres, i.e. after a manticore upgrade or on a fresh data directory
runTool(async () => {
// stashes first, the scenes sync also writes scene stashes and fails if their table doesn't exist yet
await rebuildStashes();
await rebuildScenes();
await rebuildMovies();
await rebuildActors();
console.log('Rebuilt all manticore tables');
});