DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added unextracted property to keep paginating when extracting scenes.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-10-28 01:59:53 +02:00
parent 53357d4bd2
commit 0864154a0e
3 changed files with 23 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/updates.js
View File

@@ -66,10 +66,10 @@ async function filterUniqueReleases(releases) {
return { uniqueReleases, duplicateReleases };
}
function needNextPage(pageReleases, accReleases, isUpcoming) {
function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = 0) {
const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
if (uniquePageReleases.length === 0) {
if (uniquePageReleases.length + unextracted === 0) {
// page is empty, or only contains scenes from previous page
return false;
}
@@ -78,7 +78,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming) {
return uniquePageReleases.length > 0 && argv.paginateUpcoming;
}
if (uniquePageReleases.length > 0) {
if (uniquePageReleases.length + unextracted > 0) {
if (argv.last) {
return accReleases.length + pageReleases.length < argv.last;
}
@@ -111,10 +111,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
parameters: getRecursiveParameters(entity),
};
const pageReleases = isUpcoming
const rawPageReleases = isUpcoming
? await scraper.fetchUpcoming(entity, page, options, preData)
: await scraper.fetchLatest(entity, page, options, preData);
const pageReleases = rawPageReleases.scenes || rawPageReleases;
if (!Array.isArray(pageReleases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${pageReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
@@ -128,7 +130,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
}
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming)) {
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming, rawPageReleases.unextracted)) {
return scrapeReleasesPage(page + 1, accReleases.concat(pageReleasesWithEntity), isUpcoming);
}