Added sparse date mode. Fixed profile fetch error catching. Updated Kelly Madison scraper, using site IDs and fixed profile scraper.
This commit is contained in:
@@ -147,10 +147,12 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||
|
||||
const releases = await scrapeReleasesPage(argv.page || 1, []);
|
||||
|
||||
const hasDates = releases.every((release) => !!release.date);
|
||||
const hasDates = argv.filterSparseDates
|
||||
? releases.some((release) => !!release.date)
|
||||
: releases.every((release) => !!release.date);
|
||||
|
||||
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
||||
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|
||||
|| (hasDates && releases.filter((release) => release.date && moment(release.date).isAfter(argv.after)))
|
||||
|| releases.slice(0, Math.max(isUpcoming ? argv.upcomingMissingDateLimit : argv.missingDateLimit, 0));
|
||||
|
||||
const { uniqueReleases, duplicateReleases } = argv.force
|
||||
|
||||
Reference in New Issue
Block a user