Added beforeNetwork hook, used by MindGeek. Added Filthy Kings to Gamma.

This commit is contained in:
DebaucheryLibrarian
2021-10-27 17:19:23 +02:00
parent e5ad1648eb
commit a22c4d5679
13 changed files with 25 additions and 9 deletions

View File

@@ -107,6 +107,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const options = {
...config.options[scraper.slug],
...include,
...preData,
parameters: getRecursiveParameters(entity),
};
@@ -207,7 +208,7 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
scrapeMovies(scraper, channelEntity, preData),
]);
logger.info(`Fetching ${latestReleases.uniqueReleases.length} latest and ${upcomingReleases.uniqueReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
logger.info(`Fetching ${argv.latest ? latestReleases.uniqueReleases.length : 'no'} latest and ${argv.upcoming ? upcomingReleases.uniqueReleases.length : 'no'} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return {
uniqueReleases: [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases],
@@ -215,7 +216,7 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
};
}
async function scrapeChannel(channelEntity, accNetworkReleases) {
async function scrapeChannel(channelEntity, accNetworkReleases, beforeNetwork) {
const scraper = resolveScraper(channelEntity);
const layoutScraper = resolveLayoutScraper(channelEntity, scraper);
@@ -230,6 +231,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
return await scrapeChannelReleases(layoutScraper, channelEntity, {
...accNetworkReleases,
beforeFetchLatest,
beforeNetwork,
});
} catch (error) {
logger.error(`Failed to scrape releases from ${channelEntity.name} using ${scraper.slug}: ${error.message}`);
@@ -257,10 +259,12 @@ async function scrapeNetworkSequential(networkEntity) {
}
async function scrapeNetworkParallel(networkEntity) {
const beforeNetwork = await networkEntity.scraper.beforeNetwork?.(networkEntity);
return Promise.map(
networkEntity.includedChildren,
async (channelEntity) => {
const { uniqueReleases } = await scrapeChannel(channelEntity, networkEntity);
const { uniqueReleases } = await scrapeChannel(channelEntity, null, beforeNetwork);
return uniqueReleases;
},