Added scraper for FCUK's coed sites.

This commit is contained in:
DebaucheryLibrarian
2020-07-15 04:51:39 +02:00
parent 17b3ba1272
commit c62df2228b
26 changed files with 97 additions and 11 deletions

View File

@@ -53,15 +53,15 @@ async function filterUniqueReleases(latestReleases, accReleases) {
}
function needNextPage(uniqueReleases, pageAccReleases) {
if (argv.last && pageAccReleases.length < argv.last) {
// request for last N releases not yet satisfied
return true;
}
if (uniqueReleases.length === 0) {
return false;
}
if (argv.last && pageAccReleases.length < argv.last) {
// TODO: find a way to paginate if scraper filters page with multiple channels, see Kelly Madison
return true;
}
if (uniqueReleases.every(release => !!release.date)) {
const oldestReleaseOnPage = uniqueReleases
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)