Added waitImmediate to deep scrape, reduced concurrency.
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
'use strict';
|
||||
|
||||
const util = require('util');
|
||||
const fs = require('fs').promises;
|
||||
const Promise = require('bluebird');
|
||||
const { JSDOM } = require('jsdom');
|
||||
|
||||
const waitImmediate = util.promisify(setImmediate);
|
||||
|
||||
async function init() {
|
||||
let peak = 0;
|
||||
const files = await fs.readdir('./html');
|
||||
@@ -19,9 +22,9 @@ async function init() {
|
||||
|
||||
console.log(`Memory usage: ${usage.toFixed(2)} MB, peak ${peak.toFixed(2)} MB`);
|
||||
|
||||
await Promise.delay(100);
|
||||
await waitImmediate;
|
||||
}, {
|
||||
concurrency: 10,
|
||||
concurrency: 100,
|
||||
});
|
||||
|
||||
await Promise.delay(2000);
|
||||
|
||||
Reference in New Issue
Block a user