Improved update scrape pagination and limits.

This commit is contained in:
DebaucheryLibrarian 2020-08-21 03:55:51 +02:00
parent 4ec89e2cc8
commit 1c17cd1be5
3 changed files with 54 additions and 72 deletions

View File

@ -82,8 +82,8 @@ export default {
<style lang="scss" scoped> <style lang="scss" scoped>
.home { .home {
flex-grow: 1;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
flex-grow: 1;
} }
</style> </style>

View File

@ -38,10 +38,10 @@ async function init() {
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])]; : [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean); const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean);
const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]); const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
const movieScenes = deepMovies.map(movie => movie.scenes).flat().filter(Boolean); const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean);
const deepMovieScenes = await fetchScenes(movieScenes); const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes;
if (argv.inspect) { if (argv.inspect) {
console.log(util.inspect(deepScenes)); console.log(util.inspect(deepScenes));
@ -52,7 +52,7 @@ async function init() {
if (deepMovies.length > 0) { if (deepMovies.length > 0) {
const storedMovieScenes = await storeScenes(deepMovieScenes); const storedMovieScenes = await storeScenes(deepMovieScenes);
await storeMovies(deepMovies, storedMovieScenes); await storeMovies(deepMovies, storedMovieScenes || []);
} }
if (deepScenes.length > 0 || deepMovieScenes.length > 0) { if (deepScenes.length > 0 || deepMovieScenes.length > 0) {

View File

@ -37,18 +37,16 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return uniqueReleases; return uniqueReleases;
} }
function needNextPage(uniqueReleases, pageAccReleases, hasDates) { function needNextPage(releases, uniqueReleases, totalReleases, hasDates) {
if (uniqueReleases.length === 0) { if (argv.last) {
return false; return totalReleases + releases.length < argv.last;
} }
if (argv.last && pageAccReleases.length < argv.last) { if (!hasDates) {
// TODO: find a way to paginate if scraper filters page with multiple channels, see Kelly Madison return totalReleases + releases.length < argv.nullDateLimit;
return true;
} }
if (hasDates) { const oldestReleaseOnPage = releases
const oldestReleaseOnPage = uniqueReleases
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) .sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0]; .slice(-1)[0];
@ -56,62 +54,41 @@ function needNextPage(uniqueReleases, pageAccReleases, hasDates) {
// oldest release on page is newer than the specified date cut-off // oldest release on page is newer than the specified date cut-off
return true; return true;
} }
}
// dates missing, and limit for scenes without dates not yet reached return false;
return pageAccReleases.length <= argv.nullDateLimit;
} }
async function scrapeReleases(scraper, entity, preData, upcoming = false) { async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) {
const scrapePage = async (page = 1, accReleases = []) => { const releases = upcoming
const latestReleases = upcoming
? await scraper.fetchUpcoming(entity, page, include, preData) ? await scraper.fetchUpcoming(entity, page, include, preData)
: await scraper.fetchLatest(entity, page, include, preData); : await scraper.fetchLatest(entity, page, include, preData);
if (!Array.isArray(latestReleases)) { if (!Array.isArray(releases)) {
// scraper is unable to fetch the releases and returned a HTTP code or null // scraper is unable to fetch the releases and returned a HTTP code or null
logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`); logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
return accReleases; return accReleases;
} }
const latestReleasesWithEntity = latestReleases.map(release => ({ const releasesWithEntity = releases.map(release => ({
...release, ...release,
entity: release.entity || entity, // allow override entity: release.entity || entity, // allow override
})); // attach entity the release is assigned to when stored })); // attach entity the release is assigned to when stored
const hasDates = latestReleasesWithEntity.every(release => !!release.date); const hasDates = releasesWithEntity.every(release => !!release.date);
console.log(hasDates); const limitedReleases = (argv.last && releasesWithEntity.slice(0, Math.max(argv.last - totalReleases, 0)))
|| (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after)))
|| releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0));
const uniqueReleases = argv.redownload const uniqueReleases = argv.force
? latestReleasesWithEntity ? limitedReleases
: await filterUniqueReleases(latestReleasesWithEntity, accReleases); : await filterUniqueReleases(limitedReleases, accReleases);
const pageAccReleases = accReleases.concat(uniqueReleases); if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) {
return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length);
logger.verbose(`Scraped '${entity.name}' (${entity.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
if (needNextPage(uniqueReleases, pageAccReleases, hasDates)) {
return scrapePage(page + 1, pageAccReleases);
} }
return pageAccReleases; return accReleases.concat(uniqueReleases);
};
const rawReleases = await scrapePage(argv.page || 1, []);
const releases = upcoming
? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true }))
: rawReleases;
if (argv.last) {
return releases.slice(0, argv.last);
}
if (releases.every(release => release.date)) {
return releases.filter(release => moment(release.date).isAfter(argv.after));
}
return releases.slice(0, argv.nullDateLimit);
} }
async function scrapeLatestReleases(scraper, entity, preData) { async function scrapeLatestReleases(scraper, entity, preData) {
@ -120,9 +97,12 @@ async function scrapeLatestReleases(scraper, entity, preData) {
} }
try { try {
return await scrapeReleases(scraper, entity, preData, false); return await scrapeReleases(scraper, entity, preData, false, argv.page || 1);
} catch (error) { } catch (error) {
if (argv.debug) {
console.trace(error); console.trace(error);
}
logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
} }
@ -137,6 +117,10 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
try { try {
return await scrapeReleases(scraper, entity, preData, true); return await scrapeReleases(scraper, entity, preData, true);
} catch (error) { } catch (error) {
if (argv.debug) {
console.trace(error);
}
logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
} }
@ -159,7 +143,7 @@ async function scrapeMovies(scraper, entity) {
} }
async function scrapeChannelReleases(scraper, channelEntity, preData) { async function scrapeChannelReleases(scraper, channelEntity, preData) {
const [latestReleases, upcomingReleases, movies] = await Promise.all([ const [latestReleases, upcomingReleases] = await Promise.all([
argv.latest argv.latest
? scrapeLatestReleases(scraper, channelEntity, preData) ? scrapeLatestReleases(scraper, channelEntity, preData)
: [], : [],
@ -171,8 +155,6 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
: [], : [],
]); ]);
console.log(movies);
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`); logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return [...latestReleases, ...upcomingReleases]; return [...latestReleases, ...upcomingReleases];