diff --git a/src/argv.js b/src/argv.js index bfa15d78..d65d3e26 100644 --- a/src/argv.js +++ b/src/argv.js @@ -118,6 +118,11 @@ const { argv } = yargs type: 'boolean', default: false, }) + .option('paginate-upcoming', { + describe: 'Attempt \'next\' upcoming page, usually unavailable.', + type: 'boolean', + default: false, + }) .option('movies', { describe: 'Scrape movies from channels', type: 'boolean', diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index ba1e31bf..585c14c4 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -8,6 +8,7 @@ const { JSDOM } = require('jsdom'); const moment = require('moment'); const { get, geta, ctxa, parseDate } = require('../utils/q'); +const http = require('../utils/http'); const { heightToCm } = require('../utils/convert'); const slugify = require('../utils/slugify'); @@ -360,7 +361,7 @@ async function fetchUpcoming(site) { if (site.parameters?.upcoming === false) return null; const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`; - const res = await bhttp.get(url); + const res = await http.get(url); if (res.statusCode === 200) { return scrapeUpcoming(res.body.toString(), site); diff --git a/src/updates.js b/src/updates.js index 3db6a6a1..d1290dd7 100644 --- a/src/updates.js +++ b/src/updates.js @@ -37,7 +37,11 @@ async function filterUniqueReleases(latestReleases, accReleases) { return uniqueReleases; } -function needNextPage(releases, uniqueReleases, totalReleases, hasDates) { +function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) { + if (upcoming && uniqueReleases.length > 0) { + return argv.paginateUpcoming; + } + if (argv.last) { return totalReleases + releases.length < argv.last; } @@ -46,6 +50,10 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates) { return totalReleases + releases.length < argv.nullDateLimit; } + if (uniqueReleases.length === 0) { + return false; + } + const oldestReleaseOnPage = releases .sort((releaseA, releaseB) => releaseB.date - releaseA.date) .slice(-1)[0]; @@ -84,7 +92,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page = ? limitedReleases : await filterUniqueReleases(limitedReleases, accReleases); - if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) { + if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) { return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length); }