diff --git a/assets/components/home/home.vue b/assets/components/home/home.vue index 3aee84d6..14fb0e2c 100644 --- a/assets/components/home/home.vue +++ b/assets/components/home/home.vue @@ -82,8 +82,8 @@ export default { diff --git a/src/app.js b/src/app.js index eb9aca00..e849fb89 100644 --- a/src/app.js +++ b/src/app.js @@ -38,10 +38,10 @@ async function init() { : [...(updateBaseScenes || []), ...(actorBaseScenes || [])]; const sceneMovies = deepScenes && deepScenes.map(scene => scene.movie).filter(Boolean); - const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]); + const deepMovies = argv.sceneMovies && await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]); - const movieScenes = deepMovies.map(movie => movie.scenes).flat().filter(Boolean); - const deepMovieScenes = await fetchScenes(movieScenes); + const movieScenes = deepMovies.map(movie => movie.scenes.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean); + const deepMovieScenes = argv.movieScenes ? await fetchScenes(movieScenes) : movieScenes; if (argv.inspect) { console.log(util.inspect(deepScenes)); @@ -52,7 +52,7 @@ async function init() { if (deepMovies.length > 0) { const storedMovieScenes = await storeScenes(deepMovieScenes); - await storeMovies(deepMovies, storedMovieScenes); + await storeMovies(deepMovies, storedMovieScenes || []); } if (deepScenes.length > 0 || deepMovieScenes.length > 0) { diff --git a/src/updates.js b/src/updates.js index d4536309..3db6a6a1 100644 --- a/src/updates.js +++ b/src/updates.js @@ -37,81 +37,58 @@ async function filterUniqueReleases(latestReleases, accReleases) { return uniqueReleases; } -function needNextPage(uniqueReleases, pageAccReleases, hasDates) { - if (uniqueReleases.length === 0) { - return false; +function needNextPage(releases, uniqueReleases, totalReleases, hasDates) { + if (argv.last) { + return totalReleases + releases.length < argv.last; } - if (argv.last && pageAccReleases.length < argv.last) { - // TODO: find a way to paginate if scraper filters page with multiple channels, see Kelly Madison + if (!hasDates) { + return totalReleases + releases.length < argv.nullDateLimit; + } + + const oldestReleaseOnPage = releases + .sort((releaseA, releaseB) => releaseB.date - releaseA.date) + .slice(-1)[0]; + + if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) { + // oldest release on page is newer than the specified date cut-off return true; } - if (hasDates) { - const oldestReleaseOnPage = uniqueReleases - .sort((releaseA, releaseB) => releaseB.date - releaseA.date) - .slice(-1)[0]; - - if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) { - // oldest release on page is newer than the specified date cut-off - return true; - } - } - - // dates missing, and limit for scenes without dates not yet reached - return pageAccReleases.length <= argv.nullDateLimit; + return false; } -async function scrapeReleases(scraper, entity, preData, upcoming = false) { - const scrapePage = async (page = 1, accReleases = []) => { - const latestReleases = upcoming - ? await scraper.fetchUpcoming(entity, page, include, preData) - : await scraper.fetchLatest(entity, page, include, preData); - - if (!Array.isArray(latestReleases)) { - // scraper is unable to fetch the releases and returned a HTTP code or null - logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`); - return accReleases; - } - - const latestReleasesWithEntity = latestReleases.map(release => ({ - ...release, - entity: release.entity || entity, // allow override - })); // attach entity the release is assigned to when stored - - const hasDates = latestReleasesWithEntity.every(release => !!release.date); - - console.log(hasDates); - - const uniqueReleases = argv.redownload - ? latestReleasesWithEntity - : await filterUniqueReleases(latestReleasesWithEntity, accReleases); - - const pageAccReleases = accReleases.concat(uniqueReleases); - - logger.verbose(`Scraped '${entity.name}' (${entity.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`); - - if (needNextPage(uniqueReleases, pageAccReleases, hasDates)) { - return scrapePage(page + 1, pageAccReleases); - } - - return pageAccReleases; - }; - - const rawReleases = await scrapePage(argv.page || 1, []); +async function scrapeReleases(scraper, entity, preData, upcoming = false, page = 1, accReleases = [], totalReleases = 0) { const releases = upcoming - ? rawReleases.map(rawRelease => ({ ...rawRelease, upcoming: true })) - : rawReleases; + ? await scraper.fetchUpcoming(entity, page, include, preData) + : await scraper.fetchLatest(entity, page, include, preData); - if (argv.last) { - return releases.slice(0, argv.last); + if (!Array.isArray(releases)) { + // scraper is unable to fetch the releases and returned a HTTP code or null + logger.warn(`Scraper returned ${releases} when fetching latest from '${entity.name}' (${entity.parent?.name})`); + return accReleases; } - if (releases.every(release => release.date)) { - return releases.filter(release => moment(release.date).isAfter(argv.after)); + const releasesWithEntity = releases.map(release => ({ + ...release, + entity: release.entity || entity, // allow override + })); // attach entity the release is assigned to when stored + + const hasDates = releasesWithEntity.every(release => !!release.date); + + const limitedReleases = (argv.last && releasesWithEntity.slice(0, Math.max(argv.last - totalReleases, 0))) + || (hasDates && releasesWithEntity.filter(release => moment(release.date).isAfter(argv.after))) + || releasesWithEntity.slice(0, Math.max(argv.nullDateLimit - totalReleases, 0)); + + const uniqueReleases = argv.force + ? limitedReleases + : await filterUniqueReleases(limitedReleases, accReleases); + + if (needNextPage(releases, uniqueReleases, totalReleases, hasDates)) { + return scrapeReleases(scraper, entity, preData, upcoming, page + 1, accReleases.concat(uniqueReleases), totalReleases + releases.length); } - return releases.slice(0, argv.nullDateLimit); + return accReleases.concat(uniqueReleases); } async function scrapeLatestReleases(scraper, entity, preData) { @@ -120,9 +97,12 @@ async function scrapeLatestReleases(scraper, entity, preData) { } try { - return await scrapeReleases(scraper, entity, preData, false); + return await scrapeReleases(scraper, entity, preData, false, argv.page || 1); } catch (error) { - console.trace(error); + if (argv.debug) { + console.trace(error); + } + logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); } @@ -137,6 +117,10 @@ async function scrapeUpcomingReleases(scraper, entity, preData) { try { return await scrapeReleases(scraper, entity, preData, true); } catch (error) { + if (argv.debug) { + console.trace(error); + } + logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`); } @@ -159,7 +143,7 @@ async function scrapeMovies(scraper, entity) { } async function scrapeChannelReleases(scraper, channelEntity, preData) { - const [latestReleases, upcomingReleases, movies] = await Promise.all([ + const [latestReleases, upcomingReleases] = await Promise.all([ argv.latest ? scrapeLatestReleases(scraper, channelEntity, preData) : [], @@ -171,8 +155,6 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) { : [], ]); - console.log(movies); - logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`); return [...latestReleases, ...upcomingReleases];