boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Removed window map from http module to prevent memory pile-up with update scraping.

This commit is contained in:
DebaucheryLibrarian 2021-12-04 00:32:28 +01:00
parent 883e57ce1f
commit abd063a578
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/deep.js
View File

@ -130,10 +130,10 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
if (windows.has(pathname)) { if (windows.has(pathname)) {
logger.debug(`Closing window for ${pathname}`); logger.debug(`Closing window for ${pathname}`);
}
windows.get(pathname)?.close(); windows.get(pathname).close();
windows.delete(pathname); windows.delete(pathname);
}
await waitImmediate; await waitImmediate;

4
src/utils/http.js
View File

@ -10,7 +10,7 @@ const tunnel = require('tunnel');
const Bottleneck = require('bottleneck'); const Bottleneck = require('bottleneck');
const { JSDOM, toughCookie } = require('jsdom'); const { JSDOM, toughCookie } = require('jsdom');
const windows = require('./http-windows'); // const windows = require('./http-windows');
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const virtualConsole = require('./virtual-console')(__filename); const virtualConsole = require('./virtual-console')(__filename);
@ -119,9 +119,11 @@ async function finalizeResult(res, options) {
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null; const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
const pathname = new URL(res.request.url).pathname.replace(/\//g, '_'); const pathname = new URL(res.request.url).pathname.replace(/\//g, '_');
/* unable to clear from update scrapes, nor any way to distinguish deep scrapes
if (window) { if (window) {
windows.set(pathname, window); windows.set(pathname, window);
} }
*/
if (argv.saveHtml) { if (argv.saveHtml) {
await fs.writeFile(`./html/${pathname}.html`, html); await fs.writeFile(`./html/${pathname}.html`, html);