Changed q get and geta APIs to include status, refactored scrapers. Showing front- and back-cover on movie tiles and release page (fix). Removed icons from main navigation. Returning scenes from Jules Jordan movie scraper.

This commit is contained in:
2020-03-08 04:23:10 +01:00
parent b45bb0cfbc
commit acad99bdfe
15 changed files with 222 additions and 119 deletions

View File

@@ -64,6 +64,12 @@ function qall(context, selector, attrArg, applyTrim = true) {
return Array.from(context.querySelectorAll(selector));
}
function qhtml(context, selector) {
const el = q(context, selector, null, true);
return el && el.innerHTML;
}
function qtexts(context, selector, applyTrim = true, filter = true) {
const el = q(context, selector, null, applyTrim);
if (!el) return null;
@@ -160,34 +166,36 @@ function qlength(context, selector, match, attr = 'textContent') {
const funcs = {
q,
qa: qall,
qall,
qd: qdate,
qdate,
qh: qhtml,
qhtml,
qi: qimage,
qimage,
qimages,
qposter,
qis: qimages,
ql: qlength,
qlength,
qm: qmeta,
qmeta,
qp: qposter,
qposter,
qs: qall,
qt: qtrailer,
qtext,
qtexts,
qtrailer,
qtrailers,
qurl,
qurls,
qa: qall,
qs: qall,
qd: qdate,
qi: qimage,
qis: qimages,
qp: qposter,
ql: qlength,
qm: qmeta,
qt: qtrailer,
qts: qtrailers,
qtx: qtext,
qtxt: qtext,
qtxs: qtexts,
qtxt: qtext,
qtxts: qtexts,
qu: qurl,
qurl,
qurls,
qus: qurls,
};
@@ -246,12 +254,26 @@ async function get(url, selector, headers, all = false) {
});
if (res.statusCode === 200) {
return all
const item = all
? extractAll(res.body.toString(), selector)
: extract(res.body.toString(), selector);
return {
item,
items: all ? item : [item],
res,
ok: true,
status: res.statusCode,
};
}
return null;
return {
item: null,
items: [],
res,
ok: false,
status: res.statusCode,
};
}
async function getAll(url, selector, headers) {

31
src/utils/timeout.js Normal file
View File

@@ -0,0 +1,31 @@
'use strict';
const bhttp = require('bhttp');
const sleep = 5000;
const timeout = 1000;
async function init() {
try {
const res = await bhttp.get(`https://httpstat.us/200?sleep=${sleep}`, {
responseTimeout: timeout,
});
console.log(res.statusCode);
} catch (error) {
console.log(error);
}
}
/*
/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159
err.response = response;
^
TypeError: Cannot assign to read only property 'response' of object '[object Object]'
at addErrorData (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:159:16)
at Timeout.timeoutHandler [as _onTimeout] (/home/pendulum/projectx/node_modules/bhttp/lib/bhttp.js:525:27)
*/
init();