Improved update scrape pagination and limits.

assets/components/home/home.vue

@@ -82,8 +82,8 @@ export default {
 
 <style lang="scss" scoped>
 .home {
-	flex-grow: 1;
 	display: flex;
 	flex-direction: column;
+	flex-grow: 1;
 }
 </style>

src/app.js

@@ -38,10 +38,10 @@ async function init() {
 		: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
 
 	const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean);
-	const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
+	const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
 
-	const movieScenes = deepMovies.map(movie => movie.scenes).flat().filter(Boolean);
+	const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
-	const deepMovieScenes = await fetchScenes(movieScenes);
+	const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes;
 
 	if (argv.inspect) {
 		console.log(util.inspect(deepScenes));
@@ -52,7 +52,7 @@ async function init() {
 		if (deepMovies.length > 0) {
 			const storedMovieScenes = await storeScenes(deepMovieScenes);
 
-			await storeMovies(deepMovies, storedMovieScenes);
+			await storeMovies(deepMovies, storedMovieScenes || []);
 		}
 
 		if (deepScenes.length > 0 || deepMovieScenes.length > 0) {

src/updates.js

@@ -37,81 +37,58 @@ async function filterUniqueReleases(latestReleases, accReleases) {
 	return uniqueReleases;
 }
 
-function needNextPage(uniqueReleases, pageAccReleases, hasDates) {
+function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
-	if (uniqueReleases.length === 0) {
+	if (argv.last) {
-		return false;
+		return totalReleases + releases.length < argv.last;
 	}
 
-	if (argv.last && pageAccReleases.length < argv.last) {
+	if (!hasDates) {
-		// TODO: find a way to paginate if scraper filters page with multiple channels, see Kelly Madison
+		return totalReleases + releases.length < argv.nullDateLimit;
+	}
+
+	const oldestReleaseOnPage = releases
+		.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
+		.slice(-1)[0];
+
+	if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) {
+		// oldest release on page is newer than the specified date cut-off
 		return true;
 	}
 
-	if (hasDates) {
+	return false;
-		const oldestReleaseOnPage = uniqueReleases
-			.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
-			.slice(-1)[0];
-
-		if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) {
-			// oldest release on page is newer than the specified date cut-off
-			return true;
-		}
-	}
-
-	// dates missing, and limit for scenes without dates not yet reached
-	return pageAccReleases.length <= argv.nullDateLimit;
 }
 
-async function scrapeReleases(scraper, entity, preData, upcoming = false) {
+async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) {
-	const scrapePage = async (page = 1, accReleases = []) => {
-		const latestReleases = upcoming
-			? await scraper.fetchUpcoming(entity, page, include, preData)
-			: await scraper.fetchLatest(entity, page, include, preData);
-
-		if (!Array.isArray(latestReleases)) {
-			// scraper is unable to fetch the releases and returned a HTTP code or null
-			logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
-			return accReleases;
-		}
-
-		const latestReleasesWithEntity = latestReleases.map(release => ({
-			...release,
-			entity: release.entity || entity, // allow override
-		})); // attach entity the release is assigned to when stored
-
-		const hasDates = latestReleasesWithEntity.every(release => !!release.date);
-
-		console.log(hasDates);
-
-		const uniqueReleases = argv.redownload
-			? latestReleasesWithEntity
-			: await filterUniqueReleases(latestReleasesWithEntity, accReleases);
-
-		const pageAccReleases = accReleases.concat(uniqueReleases);
-
-		logger.verbose(`Scraped '${entity.name}' (${entity.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
-
-		if (needNextPage(uniqueReleases, pageAccReleases, hasDates)) {
-			return scrapePage(page + 1, pageAccReleases);
-		}
-
-		return pageAccReleases;
-	};
-
-	const rawReleases = await scrapePage(argv.page || 1, []);
 	const releases = upcoming
-		? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true }))
+		? await scraper.fetchUpcoming(entity, page, include, preData)
-		: rawReleases;
+		: await scraper.fetchLatest(entity, page, include, preData);
 
-	if (argv.last) {
+	if (!Array.isArray(releases)) {
-		return releases.slice(0, argv.last);
+		// scraper is unable to fetch the releases and returned a HTTP code or null
+		logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
+		return accReleases;
 	}
 
-	if (releases.every(release => release.date)) {
+	const releasesWithEntity = releases.map(release => ({
-		return releases.filter(release => moment(release.date).isAfter(argv.after));
+		...release,
+		entity: release.entity || entity, // allow override
+	})); // attach entity the release is assigned to when stored
+
+	const hasDates = releasesWithEntity.every(release => !!release.date);
+
+	const limitedReleases = (argv.last && releasesWithEntity.slice(0, Math.max(argv.last - totalReleases, 0)))
+		|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
+		|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
+
+	const uniqueReleases = argv.force
+		? limitedReleases
+		: await filterUniqueReleases(limitedReleases, accReleases);
+
+	if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) {
+		return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
 	}
 
-	return releases.slice(0, argv.nullDateLimit);
+	return accReleases.concat(uniqueReleases);
 }
 
 async function scrapeLatestReleases(scraper, entity, preData) {
@@ -120,9 +97,12 @@ async function scrapeLatestReleases(scraper, entity, preData) {
 	}
 
 	try {
-		return await scrapeReleases(scraper, entity, preData, false);
+		return await scrapeReleases(scraper, entity, preData, false, argv.page || 1);
 	} catch (error) {
-		console.trace(error);
+		if (argv.debug) {
+			console.trace(error);
+		}
+
 		logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
 	}
 
@@ -137,6 +117,10 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
 	try {
 		return await scrapeReleases(scraper, entity, preData, true);
 	} catch (error) {
+		if (argv.debug) {
+			console.trace(error);
+		}
+
 		logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
 	}
 
@@ -159,7 +143,7 @@ async function scrapeMovies(scraper, entity) {
 }
 
 async function scrapeChannelReleases(scraper, channelEntity, preData) {
-	const [latestReleases, upcomingReleases, movies] = await Promise.all([
+	const [latestReleases, upcomingReleases] = await Promise.all([
 		argv.latest
 			? scrapeLatestReleases(scraper, channelEntity, preData)
 			: [],
@@ -171,8 +155,6 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
 			: [],
 	]);
 
-	console.log(movies);
-
 	logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
 
 	return [...latestReleases, ...upcomingReleases];