DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Filtering out empty or unidentified scenes from update scraper, with warning. Improved Jesse Loads Monster Facials reliability.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-10-29 15:20:59 +01:00
parent f4b1fb4831
commit b188bc5744
6 changed files with 30 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/updates.js
View File

@@ -109,7 +109,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
return accReleases;
}
const pageReleasesWithEntity = pageReleases.map(release => ({ ...release, entity: release.entity || entity }));
const validPageReleases = pageReleases.filter(release => release?.entryId); // filter out empty and unidentified releases
const pageReleasesWithEntity = validPageReleases.map(release => ({ ...release, entity: release.entity || entity }));
if (pageReleases.length > validPageReleases.length) {
logger.warn(`Found ${pageReleases.length - validPageReleases.length} empty or unidentified releases on page ${page} for '${entity.name}'`);
}
if (needNextPage(pageReleasesWithEntity, accReleases, isUpcoming)) {
return scrapeReleasesPage(page + 1, accReleases.concat(pageReleasesWithEntity), isUpcoming);
@@ -119,6 +124,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
}
const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every(release => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
@@ -133,7 +139,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
}
async function scrapeLatestReleases(scraper, entity, preData) {
if ((!argv.latest && !argv.last && !argv.after) || !scraper.fetchLatest) {
if ((!argv.latest && !argv.last) || !scraper.fetchLatest) {
return emptyReleases;
}