Not parsing HTML with jsdom when using http module directly to save memory. Added loading ellipsis to release grid pages.

This commit is contained in:
DebaucheryLibrarian
2021-10-25 02:06:24 +02:00
parent 92f9ff4104
commit 6c5d4389fe
18 changed files with 144 additions and 15 deletions

View File

@@ -23,6 +23,7 @@ Promise.config({
const defaultOptions = {
timeout: argv.requestTimeout,
encodeJSON: true,
parse: false,
headers: {
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
},
@@ -114,14 +115,14 @@ async function finalizeResult(res, options) {
if (Buffer.isBuffer(res.body)) {
const html = res.body.toString();
const window = new JSDOM(html, { virtualConsole, ...options.extract }).window;
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
return {
...res,
body: html,
html,
status: res.statusCode,
document: window.document,
document: window?.document || null,
window,
ok: res.statusCode >= 200 && res.statusCode <= 299,
};

View File

@@ -521,7 +521,11 @@ function extractAll(htmlValue, selector, options) {
async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
const res = await (method === 'post'
? http.post(urlValue, body, { ...options, headers })
: http[method](urlValue, { ...options, headers }));
: http[method](urlValue, {
...options,
headers,
parse: true,
}));
if (res.ok) {
const item = queryAll