From 8e4be06675c2132c860dc7a65aadd1b7aee51811 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sat, 18 Dec 2021 23:22:35 +0100 Subject: [PATCH] Closing JSDOM window after deep scrapes in an attempt to save memory. --- src/utils/http.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils/http.js b/src/utils/http.js index 439a218c..98e4f186 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -10,7 +10,7 @@ const tunnel = require('tunnel'); const Bottleneck = require('bottleneck'); const { JSDOM, toughCookie } = require('jsdom'); -// const windows = require('./http-windows'); +const windows = require('./http-windows'); const logger = require('../logger')(__filename); const virtualConsole = require('./virtual-console')(__filename); @@ -119,11 +119,10 @@ async function finalizeResult(res, options) { const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null; const pathname = new URL(res.request.url).pathname.replace(/\//g, '_'); - /* unable to clear from update scrapes, nor any way to distinguish deep scrapes - if (window) { + // allow window.close to be called after scraping is done, only for deep scrapes where the URL is known outside the scraper + if (window && /fetchScene|fetchMovie/.test(new Error().stack)) { windows.set(pathname, window); } - */ if (argv.saveHtml) { await fs.writeFile(`./html/${pathname}.html`, html);