DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Upgraded knex and pg versions.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-09-19 00:12:15 +02:00
parent aa265fc350
commit e78bfe4c22
3 changed files with 106 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/updates.js
View File

@@ -43,8 +43,8 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return { uniqueReleases, duplicateReleases };
}
function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
if (releases.length === 0) {
function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, upcoming) {
if (releasesOnPage.length === 0) {
return false;
}
@@ -53,15 +53,19 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin
}
if (argv.last) {
return releases.length > 0 && totalReleases + releases.length < argv.last;
// this will keep paginating until the second condition is met on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
return releasesOnPage.length > 0 && totalReleases + releasesOnPage.length < argv.last;
}
if (!hasDates) {
return totalReleases + releases.length < argv.nullDateLimit;
return totalReleases + releasesOnPage.length < argv.nullDateLimit;
}
if (argv.after) {
const oldestReleaseOnPage = releases
// this will keep paginating infinitely on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
const oldestReleaseOnPage = releasesOnPage
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0];