From 26185ba1cb1069c079b9abd79b1e5e871ba39f7d Mon Sep 17 00:00:00 2001
From: DebaucheryLibrarian <moonloop.adult@protonmail.com>
Date: Mon, 10 Feb 2025 00:13:34 +0100
Subject: [PATCH] Chunking manticore seen sync tool to prevent timeouts.

---
 src/tools/manticore-scenes.js | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/tools/manticore-scenes.js b/src/tools/manticore-scenes.js
index 1ad8fcf0..d9104ccd 100644
--- a/src/tools/manticore-scenes.js
+++ b/src/tools/manticore-scenes.js
@@ -6,6 +6,7 @@ const args = require('yargs').argv;
 const { format } = require('date-fns');
 
 const knex = require('../knex');
+const chunk = require('../utils/chunk');
 const filterTitle = require('../utils/filter-title');
 
 const mantiClient = new manticore.ApiClient();
@@ -130,9 +131,12 @@ async function init() {
 			 dupe_index int
 		)`);
 
+		console.log('Recreated scenes table');
+		console.log('Fetching scenes from primary database');
+
 		const scenes = await fetchScenes();
 
-		console.log(scenes.length, scenes.at(0));
+		console.log('Fetched scenes from primary database');
 
 		const docs = scenes.map((scene) => {
 			const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
@@ -178,9 +182,17 @@ async function init() {
 			};
 		});
 
-		const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
+		// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
+		chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
+			const acc = await chain;
+			const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
 
-		console.log('data', data);
+			console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
+
+			return acc.concat(data.items);
+		}, Promise.resolve([]));
+
+		// console.log('data', accData);
 	}
 
 	knex.destroy();