Updated Jules Jordan scraper.

This commit is contained in:
DebaucheryLibrarian
2023-07-06 00:14:38 +02:00
parent 66439b3b17
commit 0905847ffa
609 changed files with 391 additions and 277 deletions

View File

@@ -2,6 +2,7 @@
const knex = require('../knex');
const chunk = require('./chunk');
const logger = require('../logger')(__filename);
async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
if (items.length === 0) {
@@ -26,9 +27,14 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
query: knex(table).insert(chunkItems),
}).transacting(transaction));
const responses = await Promise.all(queries);
try {
const responses = await Promise.all(queries);
return responses.flat().map((response) => response.rows).flat();
return responses.flat().map((response) => response.rows).flat();
} catch (error) {
logger.error(`Failed bulk insert: ${error.message} (${error.detail})`);
throw error;
}
});
}