Changed q get and geta APIs to include status, refactored scrapers. Showing front- and back-cover on movie tiles and release page (fix). Removed icons from main navigation. Returning scenes from Jules Jordan movie scraper.
This commit is contained in:
@@ -64,6 +64,12 @@ function qall(context, selector, attrArg, applyTrim = true) {
|
||||
return Array.from(context.querySelectorAll(selector));
|
||||
}
|
||||
|
||||
function qhtml(context, selector) {
|
||||
const el = q(context, selector, null, true);
|
||||
|
||||
return el && el.innerHTML;
|
||||
}
|
||||
|
||||
function qtexts(context, selector, applyTrim = true, filter = true) {
|
||||
const el = q(context, selector, null, applyTrim);
|
||||
if (!el) return null;
|
||||
@@ -160,34 +166,36 @@ function qlength(context, selector, match, attr = 'textContent') {
|
||||
|
||||
const funcs = {
|
||||
q,
|
||||
qa: qall,
|
||||
qall,
|
||||
qd: qdate,
|
||||
qdate,
|
||||
qh: qhtml,
|
||||
qhtml,
|
||||
qi: qimage,
|
||||
qimage,
|
||||
qimages,
|
||||
qposter,
|
||||
qis: qimages,
|
||||
ql: qlength,
|
||||
qlength,
|
||||
qm: qmeta,
|
||||
qmeta,
|
||||
qp: qposter,
|
||||
qposter,
|
||||
qs: qall,
|
||||
qt: qtrailer,
|
||||
qtext,
|
||||
qtexts,
|
||||
qtrailer,
|
||||
qtrailers,
|
||||
qurl,
|
||||
qurls,
|
||||
qa: qall,
|
||||
qs: qall,
|
||||
qd: qdate,
|
||||
qi: qimage,
|
||||
qis: qimages,
|
||||
qp: qposter,
|
||||
ql: qlength,
|
||||
qm: qmeta,
|
||||
qt: qtrailer,
|
||||
qts: qtrailers,
|
||||
qtx: qtext,
|
||||
qtxt: qtext,
|
||||
qtxs: qtexts,
|
||||
qtxt: qtext,
|
||||
qtxts: qtexts,
|
||||
qu: qurl,
|
||||
qurl,
|
||||
qurls,
|
||||
qus: qurls,
|
||||
};
|
||||
|
||||
@@ -246,12 +254,26 @@ async function get(url, selector, headers, all = false) {
|
||||
});
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
return all
|
||||
const item = all
|
||||
? extractAll(res.body.toString(), selector)
|
||||
: extract(res.body.toString(), selector);
|
||||
|
||||
return {
|
||||
item,
|
||||
items: all ? item : [item],
|
||||
res,
|
||||
ok: true,
|
||||
status: res.statusCode,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
return {
|
||||
item: null,
|
||||
items: [],
|
||||
res,
|
||||
ok: false,
|
||||
status: res.statusCode,
|
||||
};
|
||||
}
|
||||
|
||||
async function getAll(url, selector, headers) {
|
||||
|
||||
31
src/utils/timeout.js
Normal file
31
src/utils/timeout.js
Normal file
@@ -0,0 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
const sleep = 5000;
|
||||
const timeout = 1000;
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
const res = await bhttp.get(`https://httpstat.us/200?sleep=${sleep}`, {
|
||||
responseTimeout: timeout,
|
||||
});
|
||||
|
||||
console.log(res.statusCode);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159
|
||||
err.response = response;
|
||||
^
|
||||
|
||||
TypeError: Cannot assign to read only property 'response' of object '[object Object]'
|
||||
at addErrorData (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159:16)
|
||||
at Timeout.timeoutHandler [as _onTimeout] (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:525:27)
|
||||
*/
|
||||
|
||||
|
||||
init();
|
||||
Reference in New Issue
Block a user