Fixed various Kelly Madison scraper issues.

This commit is contained in:
DebaucheryLibrarian
2020-12-17 02:05:01 +01:00
parent d0f8e21466
commit cd8e810c35
25 changed files with 43 additions and 32 deletions

View File

@@ -79,6 +79,10 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
const options = {
...defaultOptions,
...requestOptions,
headers: {
...defaultOptions.headers,
...requestOptions.headers,
},
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
stream: !!requestOptions.destination,
session: null,
@@ -158,7 +162,7 @@ async function head(url, options) {
}
function getSession(options) {
return bhttp.session(options);
return bhttp.session({ ...defaultOptions, ...options });
}
module.exports = {

View File

@@ -69,6 +69,10 @@ function prefixUrl(urlValue, origin, protocol = 'https') {
}
function q(context, selector, attrArg, applyTrim = true) {
if (!selector && context.nodeName === '#document') {
return null;
}
const attr = attrArg === true ? 'textContent' : attrArg;
if (attr) {