Added and updated tag photos.

src/updates.js

@@ -23,7 +23,23 @@ function mapReleasesToSiteIdAndEntryId(acc, release) {
 	return acc;
 }
 
-async function filterUniqueReleases(latestReleases, accReleases) {
+function filterLocalUniqueReleases(releases, accReleases) {
+
+}
+
+async function filterUniqueReleases(releases) {
+	const releaseIdentifiers = releases
+		.map(release => [release.entity.id, release.entryId]);
+
+	const duplicateReleaseEntries = await knex('releases')
+		.select(knex.raw('releases.*, row_to_json(entities) as entity'))
+		.leftJoin('entities', 'entities.id', 'releases.entity_id')
+		.whereIn(['entity_id', 'entry_id'], releaseIdentifiers);
+
+	const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release));
+}
+
+async function filterUniqueReleasesLegacy(latestReleases, accReleases) {
 	const latestReleaseIdentifiers = latestReleases
 		.map(release => [release.entity.id, release.entryId]);
 
@@ -56,7 +72,7 @@ async function filterUniqueReleases(latestReleases, accReleases) {
 	};
 }
 
-function needNextPage(releasesOnPage, uniqueReleasesOnPage, localUniqueReleasesOnPage, totalReleases, hasDates, upcoming) {
+function needNextPageLegacy(releasesOnPage, uniqueReleasesOnPage, localUniqueReleasesOnPage, totalReleases, hasDates, upcoming) {
 	if (releasesOnPage.length === 0) {
 		return false;
 	}
@@ -66,7 +82,7 @@ function needNextPage(releasesOnPage, uniqueReleasesOnPage, localUniqueReleasesO
 	}
 
 	// no longer works when there are no unique releases, need to keep track of /all/ releases regardless uniqueness
-	console.log(localUniqueReleasesOnPage.length);
+	// console.log(localUniqueReleasesOnPage.length);
 
 	if (localUniqueReleasesOnPage.length > 0) {
 		if (argv.last) {
@@ -94,7 +110,7 @@ function needNextPage(releasesOnPage, uniqueReleasesOnPage, localUniqueReleasesO
 	return false;
 }
 
-async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, acc = emptyReleases, totalReleases = 0) {
+async function scrapeReleasesLegacy(scraper, entity, preData, upcoming = false, page = 1, acc = emptyReleases, totalReleases = 0) {
 	const releases = upcoming
 		? await scraper.fetchUpcoming(entity, page, include, preData)
 		: await scraper.fetchLatest(entity, page, include, preData);
@@ -132,6 +148,39 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false, page =
 	return accReleases;
 }
 
+async function scrapeReleases(scraper, entity, preData, upcoming) {
+	async function scrapeReleasesPage(page, accReleases) {
+		const pageReleases = upcoming
+			? await scraper.fetchUpcoming(entity, page, include, preData)
+			: await scraper.fetchLatest(entity, page, include, preData);
+
+		if (!Array.isArray(pageReleases)) {
+			// scraper is unable to fetch the releases and returned a HTTP code or null
+			logger.warn(`Scraper returned ${pageReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
+			return accReleases;
+		}
+
+		return accReleases.concat(pageReleases);
+	}
+
+	const releases = await scrapeReleasesPage(1, emptyReleases);
+	const hasDates = releases.every(release => !!release.date);
+
+	const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
+		|| (hasDates && releases.filter(release => moment(release.date).isAfter(argv.after)))
+		|| releases.slice(0, Math.max(argv.nullDateLimit, 0));
+
+	// attach entity the release is assigned to when stored
+	const releasesWithEntity = limitedReleases.map(release => ({
+		...release,
+		entity: release.entity || entity, // allow override
+	}));
+
+	const { uniqueReleases, duplicateReleases } = argv.force
+		? { uniqueReleases: limitedReleases, localUniqueReleases: releases, duplicateReleases: [] }
+		: await filterUniqueReleases(releasesWithEntity);
+}
+
 async function scrapeLatestReleases(scraper, entity, preData) {
 	if ((!argv.latest && !argv.last && !argv.after) || !scraper.fetchLatest) {
 		return emptyReleases;