Merge branch 'experimental' into master

src/app.js

@@ -40,7 +40,7 @@ async function init() {
 	const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean);
 	const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
 
-	const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
+	const movieScenes = deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
 	const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes;
 
 	if (argv.inspect) {

src/argv.js

@@ -118,6 +118,11 @@ const { argv } = yargs
 		type: 'boolean',
 		default: false,
 	})
+	.option('paginate-upcoming', {
+		describe: 'Attempt \'next\' upcoming page, usually unavailable.',
+		type: 'boolean',
+		default: false,
+	})
 	.option('movies', {
 		describe: 'Scrape movies from channels',
 		type: 'boolean',

src/scrapers/julesjordan.js

@@ -8,6 +8,7 @@ const { JSDOM } = require('jsdom');
 const moment = require('moment');
 
 const { get, geta, ctxa, parseDate } = require('../utils/q');
+const http = require('../utils/http');
 const { heightToCm } = require('../utils/convert');
 const slugify = require('../utils/slugify');
 
@@ -360,7 +361,7 @@ async function fetchUpcoming(site) {
 	if (site.parameters?.upcoming === false) return null;
 
 	const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`;
-	const res = await bhttp.get(url);
+	const res = await http.get(url);
 
 	if (res.statusCode === 200) {
 		return scrapeUpcoming(res.body.toString(), site);

src/updates.js

@@ -37,7 +37,11 @@ async function filterUniqueReleases(latestReleases, accReleases) {
 	return uniqueReleases;
 }
 
-function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
+function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
+	if (upcoming && uniqueReleases.length > 0) {
+		return argv.paginateUpcoming;
+	}
+
 	if (argv.last) {
 		return totalReleases + releases.length < argv.last;
 	}
@@ -46,6 +50,10 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
 		return totalReleases + releases.length < argv.nullDateLimit;
 	}
 
+	if (uniqueReleases.length === 0) {
+		return false;
+	}
+
 	const oldestReleaseOnPage = releases
 		.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
 		.slice(-1)[0];
@@ -84,7 +92,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
 		? limitedReleases
 		: await filterUniqueReleases(limitedReleases, accReleases);
 
-	if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) {
+	if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) {
 		return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
 	}