From f5e3a8a2aee8be4d5d30f0383b31e44ad0bfa456 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Thu, 12 Nov 2026 02:18:13 +0100 Subject: [PATCH] Added manticore sync all tool, updated manticore driver. --- package-lock.json | 6 +- tools/manticore-actors.js | 73 ++++++++++------------ tools/manticore-movies.js | 78 ++++++++++++------------ tools/manticore-scenes.js | 122 ++++++++++++++++++------------------- tools/manticore-stashes.js | 12 ++-- tools/run-tool.js | 19 ++++++ tools/sync-all.js | 16 +++++ 7 files changed, 175 insertions(+), 151 deletions(-) create mode 100644 tools/run-tool.js create mode 100644 tools/sync-all.js diff --git a/package-lock.json b/package-lock.json index f9a69a4..401e84a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14167,9 +14167,9 @@ } }, "node_modules/manticoresearch": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.0.0.tgz", - "integrity": "sha512-a6eiAImHjhLLJgDYFRJfw2k4UL+y7ucchCbNN7q3t+6Fo4TvYf+vbS3HOcRv2h5wIFo6yaZ5N09ztKOOGqlhNw==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/manticoresearch/-/manticoresearch-10.2.0.tgz", + "integrity": "sha512-dzXZwrSVizGjV4yYjEsuzq4TQqqh9FDaCu173p7wxq2xpHwiSp5FTxiBJcMTUQa1K9Di41RUoDTMWAjUjYYjFA==", "license": "MIT", "dependencies": { "@babel/cli": "^7.0.0", diff --git a/tools/manticore-actors.js b/tools/manticore-actors.js index 92607a5..ae93716 100644 --- a/tools/manticore-actors.js +++ b/tools/manticore-actors.js @@ -1,49 +1,42 @@ import argv from '../src/argv.js'; -import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; -import redis from '../src/redis.js'; import { syncManticoreActors } from '../src/sync.js'; +import runTool from './run-tool.js'; -async function init() { - if (argv.update) { - await utilsApi.sql('drop table if exists actors'); - // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s - await utilsApi.sql(`create table actors( - id int, - name text, - aliases text, - slug string, - entity_id int, - alias_for int, - gender string, - date_of_birth bigint, - country string, - has_avatar bool, - mass int, - height int, - cup string, - natural_boobs int, - penis_length int, - penis_girth int, - stashed int, - scenes int - ) min_prefix_len = '3'`); +export async function rebuildActors() { + await utilsApi.sql('drop table if exists actors'); + // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s + await utilsApi.sql(`create table actors( + id int, + name text, + aliases text, + slug string, + entity_id int, + alias_for int, + gender string, + date_of_birth bigint, + country string, + has_avatar bool, + mass int, + height int, + cup string, + natural_boobs int, + penis_length int, + penis_girth int, + stashed int, + scenes int + ) min_prefix_len = '3'`); - console.log('Recreated actors table, syncing actors...'); + console.log('Recreated actors table, syncing actors...'); - const data = await syncManticoreActors(); - - console.log('data', data); - - await knex.destroy(); - await redis.quit(); // sync imports connect to redis, which keeps the process alive - - return; - } - - await knex.destroy(); - await redis.quit(); // sync imports connect to redis, which keeps the process alive + await syncManticoreActors(); } -init(); +if (import.meta.main) { + runTool(async () => { + if (argv.update) { + await rebuildActors(); + } + }); +} diff --git a/tools/manticore-movies.js b/tools/manticore-movies.js index 08e78f5..db8f0d9 100644 --- a/tools/manticore-movies.js +++ b/tools/manticore-movies.js @@ -1,48 +1,46 @@ import argv from '../src/argv.js'; -import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; -import redis from '../src/redis.js'; import { syncManticoreMovies } from '../src/sync.js'; +import runTool from './run-tool.js'; -async function init() { - if (argv.update) { - await utilsApi.sql('drop table if exists movies'); - // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s - await utilsApi.sql(`create table movies ( - id int, - title text, - title_filtered text, - channel_id int, - channel_name text, - channel_slug text, - network_id int, - network_name text, - network_slug text, - entity_ids multi, - actor_ids multi, - actors text, - tag_ids multi, - tags text, - meta text, - date bigint, - has_cover bool, - created_at bigint, - effective_date bigint, - stashed int, - stashed_scenes int, - stashed_total int, - dupe_index int - )`); +export async function rebuildMovies() { + await utilsApi.sql('drop table if exists movies'); + // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s + await utilsApi.sql(`create table movies ( + id int, + title text, + title_filtered text, + channel_id int, + channel_name text, + channel_slug text, + network_id int, + network_name text, + network_slug text, + entity_ids multi, + actor_ids multi, + actors text, + tag_ids multi, + tags text, + meta text, + date bigint, + has_cover bool, + created_at bigint, + effective_date bigint, + stashed int, + stashed_scenes int, + stashed_total int, + dupe_index int + )`); - console.log('Recreated movies tables, syncing movies...'); + console.log('Recreated movies tables, syncing movies...'); - const data = await syncManticoreMovies(); - - console.log('data', data); - } - - await knex.destroy(); - await redis.quit(); // sync imports connect to redis, which keeps the process alive + await syncManticoreMovies(); } -init(); +if (import.meta.main) { + runTool(async () => { + if (argv.update) { + await rebuildMovies(); + } + }); +} diff --git a/tools/manticore-scenes.js b/tools/manticore-scenes.js index 11f1f4a..36c7e72 100644 --- a/tools/manticore-scenes.js +++ b/tools/manticore-scenes.js @@ -1,71 +1,71 @@ import argv from '../src/argv.js'; -import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; -import redis from '../src/redis.js'; import { syncManticoreScenes } from '../src/sync.js'; +import runTool from './run-tool.js'; -async function init() { - if (argv.update) { - await utilsApi.sql('drop table if exists scenes'); - // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s - await utilsApi.sql(`create table scenes ( - id int, - title text, - title_filtered text, - entry_id text, - shoot_id text, - channel_id int, - channel_name text, - channel_slug text, - network_id int, - network_name text, - network_slug text, - studio_id int, - studio_name text, - studio_slug text, - entity_ids multi, - actor_ids multi, - actors text, - actor_count int, - female_count int, - male_count int, - trans_count int, - unidentified_count int, - tag_ids multi, - assigned_tag_ids multi64, - tags text, - movie_ids multi, - movies text, - serie_ids multi, - series text, - meta text, - date bigint, - fingerprints text, - is_showcased bool, - created_at bigint, - effective_date bigint, - stashed int, - dupe_index int - )`); +export async function rebuildScenes() { + await utilsApi.sql('drop table if exists scenes'); + // bigint rather than timestamp, which is unsigned 32-bit and wraps dates before 1970 into the 2100s + await utilsApi.sql(`create table scenes ( + id int, + title text, + title_filtered text, + entry_id text, + shoot_id text, + channel_id int, + channel_name text, + channel_slug text, + network_id int, + network_name text, + network_slug text, + studio_id int, + studio_name text, + studio_slug text, + entity_ids multi, + actor_ids multi, + actors text, + actor_count int, + female_count int, + male_count int, + trans_count int, + unidentified_count int, + tag_ids multi, + assigned_tag_ids multi64, + tags text, + movie_ids multi, + movies text, + serie_ids multi, + series text, + meta text, + date bigint, + fingerprints text, + is_showcased bool, + created_at bigint, + effective_date bigint, + stashed int, + dupe_index int + )`); - await utilsApi.sql('drop table if exists scenes_tags'); + await utilsApi.sql('drop table if exists scenes_tags'); - /* legacy, using packed decimal keys now - await utilsApi.sql(`create table scenes_tags ( - id int, - scene_id int, - tag_id int, - actor_id int - )`); - */ + /* legacy, using packed decimal keys now + await utilsApi.sql(`create table scenes_tags ( + id int, + scene_id int, + tag_id int, + actor_id int + )`); + */ - console.log('Recreated scenes tables, syncing scenes...'); + console.log('Recreated scenes tables, syncing scenes...'); - await syncManticoreScenes(); - } - - await knex.destroy(); - await redis.quit(); // sync imports connect to redis, which keeps the process alive + await syncManticoreScenes(); } -init(); +if (import.meta.main) { + runTool(async () => { + if (argv.update) { + await rebuildScenes(); + } + }); +} diff --git a/tools/manticore-stashes.js b/tools/manticore-stashes.js index 99bf0a5..5e1c557 100644 --- a/tools/manticore-stashes.js +++ b/tools/manticore-stashes.js @@ -1,9 +1,8 @@ -import { knexOwner as knex } from '../src/knex.js'; import { utilsApi } from '../src/manticore.js'; -import redis from '../src/redis.js'; import { syncStashes } from '../src/sync.js'; +import runTool from './run-tool.js'; -async function init() { +export async function rebuildStashes() { await utilsApi.sql('drop table if exists scenes_stashed'); await utilsApi.sql(`create table if not exists scenes_stashed ( @@ -36,9 +35,8 @@ async function init() { await syncStashes('scene'); await syncStashes('actor'); await syncStashes('movie'); - - await knex.destroy(); - await redis.quit(); // sync imports connect to redis, which keeps the process alive } -init(); +if (import.meta.main) { + runTool(rebuildStashes); +} diff --git a/tools/run-tool.js b/tools/run-tool.js new file mode 100644 index 0000000..cf8d29e --- /dev/null +++ b/tools/run-tool.js @@ -0,0 +1,19 @@ +import process from 'node:process'; + +import { knexOwner as knex } from '../src/knex.js'; +import redis from '../src/redis.js'; + +// runs a command line tool and closes the connections afterwards, the knex pool and redis would otherwise keep the process alive +export default async function runTool(tool) { + try { + await tool(); + + await knex.destroy(); + await redis.quit(); + } + catch (error) { + // the manticore client rejects with a plain response object rather than an Error, print the server's message + console.error(error instanceof Error ? error : error?.body?.error || error?.body || error); + process.exit(1); + } +} diff --git a/tools/sync-all.js b/tools/sync-all.js new file mode 100644 index 0000000..2aeee61 --- /dev/null +++ b/tools/sync-all.js @@ -0,0 +1,16 @@ +import { rebuildActors } from './manticore-actors.js'; +import { rebuildMovies } from './manticore-movies.js'; +import { rebuildScenes } from './manticore-scenes.js'; +import { rebuildStashes } from './manticore-stashes.js'; +import runTool from './run-tool.js'; + +// recreates and fills every manticore table from postgres, i.e. after a manticore upgrade or on a fresh data directory +runTool(async () => { + // stashes first, the scenes sync also writes scene stashes and fails if their table doesn't exist yet + await rebuildStashes(); + await rebuildScenes(); + await rebuildMovies(); + await rebuildActors(); + + console.log('Rebuilt all manticore tables'); +});