boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Chunking manticore seen sync tool to prevent timeouts.

This commit is contained in:
DebaucheryLibrarian 2025-02-10 00:13:34 +01:00
parent 0063c55e34
commit 26185ba1cb
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/tools/manticore-scenes.js
View File

@ -6,6 +6,7 @@ const args = require('yargs').argv;
const { format } = require('date-fns'); const { format } = require('date-fns');
const knex = require('../knex'); const knex = require('../knex');
const chunk = require('../utils/chunk');
const filterTitle = require('../utils/filter-title'); const filterTitle = require('../utils/filter-title');
const mantiClient = new manticore.ApiClient(); const mantiClient = new manticore.ApiClient();
@ -130,9 +131,12 @@ async function init() {
dupe_index int dupe_index int
)`); )`);
console.log('Recreated scenes table');
console.log('Fetching scenes from primary database');
const scenes = await fetchScenes(); const scenes = await fetchScenes();
console.log(scenes.length, scenes.at(0)); console.log('Fetched scenes from primary database');
const docs = scenes.map((scene) => { const docs = scenes.map((scene) => {
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc. const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
@ -178,9 +182,17 @@ async function init() {
}; };
}); });
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')); // const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
const acc = await chain;
const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
console.log('data', data); console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
return acc.concat(data.items);
}, Promise.resolve([]));
// console.log('data', accData);
} }
knex.destroy(); knex.destroy();