DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Using batch insert module for media, calculating chunk size based on item size.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-03-15 21:33:14 +01:00
parent 31aa1118e7
commit 1b6aaafe10
2 changed files with 24 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/utils/batch-insert.js
View File

@@ -4,11 +4,12 @@ const knex = require('../knex');
const chunk = require('./chunk');
const logger = require('../logger')(__filename);
const chunkTarget = 50_000; // PostgreSQL allows 65,535 binding parameters, allow for a bit of margin
// improved version of bulkInsert
async function batchInsert(table, items, {
conflict = true,
update = false,
chunkSize = 1000,
concurrent = false,
transaction,
commit = false,
@@ -17,6 +18,10 @@ async function batchInsert(table, items, {
throw new Error('No table specified for batch insert');
}
if (conflict && update) {
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
}
if (!Array.isArray(items)) {
throw new Error('Batch insert items are not an array');
}
@@ -25,8 +30,20 @@ async function batchInsert(table, items, {
return [];
}
const chunks = chunk(items, chunkSize);
// PostgreSQL's bindings limit applies to individual values, so item size needs to be taken into account
const itemSize = items.reduce((acc, item) => Math.max(acc, Object.keys(item).length), 0);
if (itemSize === 0) {
throw new Error('Batch insert items are empty');
}
const chunks = chunk(items, Math.floor(chunkTarget / itemSize));
const conflicts = [].concat(conflict).filter((column) => typeof column === 'string'); // conflict might be 'true'
if (conflicts.length > 0 && !update) {
throw new Error('Batch insert conflict columns must be specified together with update');
}
const trx = transaction || await knex.transaction();
try {
@@ -49,12 +66,6 @@ async function batchInsert(table, items, {
.onConflict(conflicts)
.merge();
}
throw new Error('Batch insert conflict columns must be specified together with update');
}
if (conflict && update) {
throw new Error('Batch insert conflict must specify columns, or update must be disabled');
}
// error on any conflict