boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Replaced bhttp with patched fork. Improved Jesse Loads Monster Facials scraper reliability (WIP). Added various tag photos.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-10-30 17:37:10 +01:00
parent 4af7597441
commit 39f8c037a5
43 changed files with 128 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/utils/bulk-insert.js
View File

@@ -21,12 +21,10 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
return knex.transaction(async (transaction) => {
const chunked = chunk(items, chunkSize);
// console.log(items.length, chunkSize, chunked.length, chunked[0]?.length);
const queries = chunked
.map(chunkItems => knex.raw(updated || ':query RETURNING *;', {
query: knex(table).insert(chunkItems).transacting(transaction),
}));
query: knex(table).insert(chunkItems),
}).transacting(transaction));
const responses = await Promise.all(queries);

2
src/utils/http.js
View File

@@ -4,7 +4,7 @@ const util = require('util');
const stream = require('stream');
const config = require('config');
const tunnel = require('tunnel');
const bhttp = require('bhttp');
const bhttp = require('@thependulum/bhttp');
const taskQueue = require('promise-task-queue');
const pipeline = util.promisify(stream.pipeline);