Added waitImmediate to deep scrape, reduced concurrency.

This commit is contained in:
DebaucheryLibrarian
2021-12-01 23:30:10 +01:00
parent 56a7fb0ad9
commit e41f9fa937
2 changed files with 11 additions and 3 deletions

View File

@@ -1,9 +1,12 @@
'use strict';
const util = require('util');
const fs = require('fs').promises;
const Promise = require('bluebird');
const { JSDOM } = require('jsdom');
const waitImmediate = util.promisify(setImmediate);
async function init() {
let peak = 0;
const files = await fs.readdir('./html');
@@ -19,9 +22,9 @@ async function init() {
console.log(`Memory usage: ${usage.toFixed(2)} MB, peak ${peak.toFixed(2)} MB`);
await Promise.delay(100);
await waitImmediate;
}, {
concurrency: 10,
concurrency: 100,
});
await Promise.delay(2000);