DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Merge branch 'experimental' into master

This commit is contained in:
DebaucheryLibrarian 2020-08-22 01:57:32 +02:00
parent ec3ea892af 5f5c48ea05
commit 9361f6bc53
4 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/app.js
View File

@ -40,7 +40,7 @@ async function init() {
const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean); const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]); const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean); const movieScenes = deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes; const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes;
if (argv.inspect) { if (argv.inspect) {

5
src/argv.js
View File

@ -118,6 +118,11 @@ const { argv } = yargs
type: 'boolean', type: 'boolean',
default: false, default: false,
}) })
.option('paginate-upcoming', {
describe: 'Attempt \'next\' upcoming page, usually unavailable.',
type: 'boolean',
default: false,
})
.option('movies', { .option('movies', {
describe: 'Scrape movies from channels', describe: 'Scrape movies from channels',
type: 'boolean', type: 'boolean',

3
src/scrapers/julesjordan.js
View File

@ -8,6 +8,7 @@ const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
const { get, geta, ctxa, parseDate } = require('../utils/q'); const { get, geta, ctxa, parseDate } = require('../utils/q');
const http = require('../utils/http');
const { heightToCm } = require('../utils/convert'); const { heightToCm } = require('../utils/convert');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@ -360,7 +361,7 @@ async function fetchUpcoming(site) {
if (site.parameters?.upcoming === false) return null; if (site.parameters?.upcoming === false) return null;
const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`; const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`;
const res = await bhttp.get(url); const res = await http.get(url);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeUpcoming(res.body.toString(), site); return scrapeUpcoming(res.body.toString(), site);

12
src/updates.js
View File

@ -37,7 +37,11 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return uniqueReleases; return uniqueReleases;
} }
function needNextPage(releases, uniqueReleases, totalReleases, hasDates) { function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
if (upcoming && uniqueReleases.length > 0) {
return argv.paginateUpcoming;
}
if (argv.last) { if (argv.last) {
return totalReleases + releases.length < argv.last; return totalReleases + releases.length < argv.last;
} }
@ -46,6 +50,10 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
return totalReleases + releases.length < argv.nullDateLimit; return totalReleases + releases.length < argv.nullDateLimit;
} }
if (uniqueReleases.length === 0) {
return false;
}
const oldestReleaseOnPage = releases const oldestReleaseOnPage = releases
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) .sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0]; .slice(-1)[0];
@ -84,7 +92,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
? limitedReleases ? limitedReleases
: await filterUniqueReleases(limitedReleases, accReleases); : await filterUniqueReleases(limitedReleases, accReleases);
if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) { if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) {
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length); return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
} }