DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Returning duplicate releases from pagination.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-22 04:22:56 +02:00
parent 5f5c48ea05
commit 3a5ea3dd9a
5 changed files with 46 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/updates.js
View File

@@ -10,6 +10,8 @@ const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers');
const { fetchIncludedEntities } = require('./entities');
const emptyReleases = { uniqueReleases: [], duplicateReleases: [] };
async function filterUniqueReleases(latestReleases, accReleases) {
const latestReleaseIdentifiers = latestReleases
.map(release => [release.entity.id, release.entryId]);
@@ -31,10 +33,9 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return acc;
}, {});
const uniqueReleases = latestReleases
.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
const uniqueReleases = latestReleases.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
return uniqueReleases;
return { uniqueReleases, duplicateReleases };
}
function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
@@ -66,7 +67,7 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin
return false;
}
async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) {
async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, acc = emptyReleases, totalReleases = 0) {
const releases = upcoming
? await scraper.fetchUpcoming(entity, page, include, preData)
: await scraper.fetchLatest(entity, page, include, preData);
@@ -74,7 +75,7 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
if (!Array.isArray(releases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
return accReleases;
return acc;
}
const releasesWithEntity = releases.map(release => ({
@@ -88,20 +89,25 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
const uniqueReleases = argv.force
? limitedReleases
: await filterUniqueReleases(limitedReleases, accReleases);
const { uniqueReleases, duplicateReleases } = argv.force
? { uniqueReleases: limitedReleases, duplicateReleases: [] }
: await filterUniqueReleases(limitedReleases, acc.uniqueReleases);
const accReleases = {
uniqueReleases: acc.uniqueReleases.concat(uniqueReleases),
duplicateReleases: acc.duplicateReleases.concat(duplicateReleases),
};
if (needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming)) {
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases, totalReleases + releases.length);
}
return accReleases.concat(uniqueReleases);
return accReleases;
}
async function scrapeLatestReleases(scraper, entity, preData) {
if (!scraper.fetchLatest) {
return [];
if (!argv.latest || !scraper.fetchLatest) {
return emptyReleases;
}
try {
@@ -114,12 +120,12 @@ async function scrapeLatestReleases(scraper, entity, preData) {
logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
}
return [];
return emptyReleases;
}
async function scrapeUpcomingReleases(scraper, entity, preData) {
if (!scraper.fetchUpcoming) {
return [];
if (!argv.upcoming || !scraper.fetchUpcoming) {
return emptyReleases;
}
try {
@@ -132,11 +138,11 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
}
return [];
return emptyReleases;
}
async function scrapeMovies(scraper, entity) {
if (!scraper.fetchMovies) {
if (!argv.movies || !scraper.fetchMovies) {
return [];
}
@@ -152,20 +158,16 @@ async function scrapeMovies(scraper, entity) {
async function scrapeChannelReleases(scraper, channelEntity, preData) {
const [latestReleases, upcomingReleases] = await Promise.all([
argv.latest
? scrapeLatestReleases(scraper, channelEntity, preData)
: [],
argv.upcoming
? scrapeUpcomingReleases(scraper, channelEntity, preData)
: [],
argv.movies
? scrapeMovies(scraper, channelEntity, preData)
: [],
scrapeLatestReleases(scraper, channelEntity, preData),
scrapeUpcomingReleases(scraper, channelEntity, preData),
scrapeMovies(scraper, channelEntity, preData),
]);
console.log(latestReleases);
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return [...latestReleases, ...upcomingReleases];
return [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases];
}
async function scrapeChannel(channelEntity, accNetworkReleases) {