Fixed various Kelly Madison scraper issues.
This commit is contained in:
@@ -79,6 +79,10 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
|
||||
const options = {
|
||||
...defaultOptions,
|
||||
...requestOptions,
|
||||
headers: {
|
||||
...defaultOptions.headers,
|
||||
...requestOptions.headers,
|
||||
},
|
||||
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
|
||||
stream: !!requestOptions.destination,
|
||||
session: null,
|
||||
@@ -158,7 +162,7 @@ async function head(url, options) {
|
||||
}
|
||||
|
||||
function getSession(options) {
|
||||
return bhttp.session(options);
|
||||
return bhttp.session({ ...defaultOptions, ...options });
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -69,6 +69,10 @@ function prefixUrl(urlValue, origin, protocol = 'https') {
|
||||
}
|
||||
|
||||
function q(context, selector, attrArg, applyTrim = true) {
|
||||
if (!selector && context.nodeName === '#document') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attr = attrArg === true ? 'textContent' : attrArg;
|
||||
|
||||
if (attr) {
|
||||
|
||||
Reference in New Issue
Block a user