Added beforeNetwork hook, used by MindGeek. Added Filthy Kings to Gamma.
This commit is contained in:
@@ -107,6 +107,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||
const options = {
|
||||
...config.options[scraper.slug],
|
||||
...include,
|
||||
...preData,
|
||||
parameters: getRecursiveParameters(entity),
|
||||
};
|
||||
|
||||
@@ -207,7 +208,7 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
|
||||
scrapeMovies(scraper, channelEntity, preData),
|
||||
]);
|
||||
|
||||
logger.info(`Fetching ${latestReleases.uniqueReleases.length} latest and ${upcomingReleases.uniqueReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
|
||||
logger.info(`Fetching ${argv.latest ? latestReleases.uniqueReleases.length : 'no'} latest and ${argv.upcoming ? upcomingReleases.uniqueReleases.length : 'no'} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
|
||||
|
||||
return {
|
||||
uniqueReleases: [...latestReleases.uniqueReleases, ...upcomingReleases.uniqueReleases],
|
||||
@@ -215,7 +216,7 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
|
||||
};
|
||||
}
|
||||
|
||||
async function scrapeChannel(channelEntity, accNetworkReleases) {
|
||||
async function scrapeChannel(channelEntity, accNetworkReleases, beforeNetwork) {
|
||||
const scraper = resolveScraper(channelEntity);
|
||||
const layoutScraper = resolveLayoutScraper(channelEntity, scraper);
|
||||
|
||||
@@ -230,6 +231,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
|
||||
return await scrapeChannelReleases(layoutScraper, channelEntity, {
|
||||
...accNetworkReleases,
|
||||
beforeFetchLatest,
|
||||
beforeNetwork,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(`Failed to scrape releases from ${channelEntity.name} using ${scraper.slug}: ${error.message}`);
|
||||
@@ -257,10 +259,12 @@ async function scrapeNetworkSequential(networkEntity) {
|
||||
}
|
||||
|
||||
async function scrapeNetworkParallel(networkEntity) {
|
||||
const beforeNetwork = await networkEntity.scraper.beforeNetwork?.(networkEntity);
|
||||
|
||||
return Promise.map(
|
||||
networkEntity.includedChildren,
|
||||
async (channelEntity) => {
|
||||
const { uniqueReleases } = await scrapeChannel(channelEntity, networkEntity);
|
||||
const { uniqueReleases } = await scrapeChannel(channelEntity, null, beforeNetwork);
|
||||
|
||||
return uniqueReleases;
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user