Added sparse date mode. Fixed profile fetch error catching. Updated Kelly Madison scraper, using site IDs and fixed profile scraper.

This commit is contained in:
DebaucheryLibrarian
2024-12-30 01:57:26 +01:00
parent ffea90b5e8
commit 23f76fd1be
9 changed files with 73 additions and 30 deletions

View File

@@ -147,10 +147,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const releases = await scrapeReleasesPage(argv.page || 1, []);
const hasDates = releases.every((release) => !!release.date);
const hasDates = argv.filterSparseDates
? releases.some((release) => !!release.date)
: releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| (hasDates && releases.filter((release) => release.date && moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(isUpcoming ? argv.upcomingMissingDateLimit : argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force