Refactored various modules for entities. Updated and refactored Kink scraper.
This commit is contained in:
@@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const scrapers = require('./scrapers/scrapers');
|
||||
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./entities');
|
||||
const { fetchChannelsFromArgv, fetchChannelsFromConfig } = require('./entities');
|
||||
|
||||
const afterDate = (() => {
|
||||
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
|
||||
@@ -27,7 +27,7 @@ const afterDate = (() => {
|
||||
|
||||
async function filterUniqueReleases(latestReleases, accReleases) {
|
||||
const latestReleaseIdentifiers = latestReleases
|
||||
.map(release => [release.site.id, release.entryId]);
|
||||
.map(release => [release.entity.id, release.entryId]);
|
||||
|
||||
const duplicateReleases = await knex('releases')
|
||||
.whereIn(['entity_id', 'entry_id'], latestReleaseIdentifiers);
|
||||
@@ -37,17 +37,17 @@ async function filterUniqueReleases(latestReleases, accReleases) {
|
||||
const duplicateReleasesSiteIdAndEntryIds = duplicateReleases
|
||||
.concat(accReleases)
|
||||
.reduce((acc, release) => {
|
||||
const siteId = release.entity_id || release.site.id;
|
||||
const entityId = release.entity_id || release.entity.id;
|
||||
const entryId = release.entry_id || release.entryId;
|
||||
|
||||
if (!acc[siteId]) acc[siteId] = {};
|
||||
acc[siteId][entryId] = true;
|
||||
if (!acc[entityId]) acc[entityId] = {};
|
||||
acc[entityId][entryId] = true;
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const uniqueReleases = latestReleases
|
||||
.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.site.id]?.[release.entryId]);
|
||||
.filter(release => !duplicateReleasesSiteIdAndEntryIds[release.entity.id]?.[release.entryId]);
|
||||
|
||||
return uniqueReleases;
|
||||
}
|
||||
@@ -77,15 +77,15 @@ function needNextPage(uniqueReleases, pageAccReleases) {
|
||||
return pageAccReleases.length <= argv.nullDateLimit;
|
||||
}
|
||||
|
||||
async function scrapeReleases(scraper, site, preData, upcoming = false) {
|
||||
async function scrapeReleases(scraper, entity, preData, upcoming = false) {
|
||||
const scrapePage = async (page = 1, accReleases = []) => {
|
||||
const latestReleases = upcoming
|
||||
? await scraper.fetchUpcoming(site, page, preData, include)
|
||||
: await scraper.fetchLatest(site, page, preData, include);
|
||||
? await scraper.fetchUpcoming(entity, page, preData, include)
|
||||
: await scraper.fetchLatest(entity, page, preData, include);
|
||||
|
||||
if (!Array.isArray(latestReleases)) {
|
||||
// scraper is unable to fetch the releases and returned a HTTP code or null
|
||||
logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${site.name}' (${site.parent?.name})`);
|
||||
logger.warn(`Scraper returned ${latestReleases} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
|
||||
return accReleases;
|
||||
}
|
||||
|
||||
@@ -94,15 +94,15 @@ async function scrapeReleases(scraper, site, preData, upcoming = false) {
|
||||
return accReleases;
|
||||
}
|
||||
|
||||
const latestReleasesWithSite = latestReleases.map(release => ({ ...release, site: release.site || site })); // attach site release is assigned to when stored
|
||||
const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity })); // attach entity the release is assigned to when stored
|
||||
|
||||
const uniqueReleases = argv.redownload
|
||||
? latestReleasesWithSite
|
||||
: await filterUniqueReleases(latestReleasesWithSite, accReleases);
|
||||
? latestReleasesWithEntity
|
||||
: await filterUniqueReleases(latestReleasesWithEntity, accReleases);
|
||||
|
||||
const pageAccReleases = accReleases.concat(uniqueReleases);
|
||||
|
||||
logger.verbose(`Scraped '${site.name}' (${site.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
|
||||
logger.verbose(`Scraped '${entity.name}' (${entity.parent?.name}) ${upcoming ? 'upcoming' : 'latest'} page ${page}, found ${uniqueReleases.length} unique updates`);
|
||||
|
||||
if (needNextPage(uniqueReleases, pageAccReleases)) {
|
||||
return scrapePage(page + 1, pageAccReleases);
|
||||
@@ -127,70 +127,70 @@ async function scrapeReleases(scraper, site, preData, upcoming = false) {
|
||||
return releases.slice(0, argv.nullDateLimit);
|
||||
}
|
||||
|
||||
async function scrapeLatestReleases(scraper, site, preData) {
|
||||
async function scrapeLatestReleases(scraper, entity, preData) {
|
||||
if (!scraper.fetchLatest) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
return await scrapeReleases(scraper, site, preData, false);
|
||||
return await scrapeReleases(scraper, entity, preData, false);
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to scrape latest updates for '${site.slug}' (${site.parent?.slug}): ${error.message}`);
|
||||
logger.warn(`Failed to scrape latest updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function scrapeUpcomingReleases(scraper, site, preData) {
|
||||
async function scrapeUpcomingReleases(scraper, entity, preData) {
|
||||
if (!scraper.fetchUpcoming) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
return await scrapeReleases(scraper, site, preData, true);
|
||||
return await scrapeReleases(scraper, entity, preData, true);
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to scrape upcoming updates for '${site.slug}' (${site.parent?.slug}): ${error.message}`);
|
||||
logger.warn(`Failed to scrape upcoming updates for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
async function scrapeSiteReleases(scraper, site, preData) {
|
||||
async function scrapeChannelReleases(scraper, channelEntity, preData) {
|
||||
const [latestReleases, upcomingReleases] = await Promise.all([
|
||||
argv.latest
|
||||
? scrapeLatestReleases(scraper, site, preData)
|
||||
? scrapeLatestReleases(scraper, channelEntity, preData)
|
||||
: [],
|
||||
argv.upcoming
|
||||
? scrapeUpcomingReleases(scraper, site, preData)
|
||||
? scrapeUpcomingReleases(scraper, channelEntity, preData)
|
||||
: [],
|
||||
]);
|
||||
|
||||
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${site.name}' (${site.parent.name})`);
|
||||
logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent.name})`);
|
||||
|
||||
return [...latestReleases, ...upcomingReleases];
|
||||
}
|
||||
|
||||
async function scrapeSite(site, accSiteReleases) {
|
||||
const scraper = scrapers.releases[site.slug]
|
||||
|| scrapers.releases[site.parent?.slug]
|
||||
|| scrapers.releases[site.parent?.parent?.slug];
|
||||
async function scrapeChannel(channelEntity, accNetworkReleases) {
|
||||
const scraper = scrapers.releases[channelEntity.slug]
|
||||
|| scrapers.releases[channelEntity.parent?.slug]
|
||||
|| scrapers.releases[channelEntity.parent?.parent?.slug];
|
||||
|
||||
if (!scraper) {
|
||||
logger.warn(`No scraper found for '${site.name}' (${site.parent.name})`);
|
||||
logger.warn(`No scraper found for '${channelEntity.name}' (${channelEntity.parent.name})`);
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const beforeFetchLatest = await scraper.beforeFetchLatest?.(site);
|
||||
const beforeFetchLatest = await scraper.beforeFetchLatest?.(channelEntity);
|
||||
|
||||
const siteReleases = await scrapeSiteReleases(scraper, site, {
|
||||
accSiteReleases,
|
||||
const channelEntityReleases = await scrapeChannelReleases(scraper, channelEntity, {
|
||||
accNetworkReleases,
|
||||
beforeFetchLatest,
|
||||
});
|
||||
|
||||
return siteReleases.map(release => ({ ...release, site }));
|
||||
return channelEntityReleases.map(release => ({ ...release, channelEntity }));
|
||||
} catch (error) {
|
||||
logger.error(`Failed to scrape releases from ${site.name} using ${scraper.slug}: ${error.message}`);
|
||||
logger.error(`Failed to scrape releases from ${channelEntity.name} using ${scraper.slug}: ${error.message}`);
|
||||
|
||||
return [];
|
||||
}
|
||||
@@ -199,11 +199,11 @@ async function scrapeSite(site, accSiteReleases) {
|
||||
async function scrapeNetworkSequential(networkEntity) {
|
||||
return Promise.reduce(
|
||||
networkEntity.children,
|
||||
async (chain, siteEntity) => {
|
||||
const accSiteReleases = await chain;
|
||||
const siteReleases = await scrapeSite(siteEntity, networkEntity, accSiteReleases);
|
||||
async (chain, channelEntity) => {
|
||||
const accNetworkReleases = await chain;
|
||||
const channelReleases = await scrapeChannel(channelEntity, networkEntity, accNetworkReleases);
|
||||
|
||||
return accSiteReleases.concat(siteReleases);
|
||||
return accNetworkReleases.concat(channelReleases);
|
||||
},
|
||||
Promise.resolve([]),
|
||||
);
|
||||
@@ -212,21 +212,21 @@ async function scrapeNetworkSequential(networkEntity) {
|
||||
async function scrapeNetworkParallel(networkEntity) {
|
||||
return Promise.map(
|
||||
networkEntity.children,
|
||||
async siteEntity => scrapeSite(siteEntity, networkEntity),
|
||||
async channelEntity => scrapeChannel(channelEntity, networkEntity),
|
||||
{ concurrency: 3 },
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchUpdates() {
|
||||
const includedNetworks = argv.sites || argv.networks || argv.from
|
||||
? await fetchSitesFromArgv()
|
||||
: await fetchSitesFromConfig();
|
||||
const includedNetworks = argv.channels || argv.networks
|
||||
? await fetchChannelsFromArgv()
|
||||
: await fetchChannelsFromConfig();
|
||||
|
||||
const scrapedNetworks = await Promise.map(
|
||||
includedNetworks,
|
||||
async network => (network.parameters?.sequential
|
||||
? scrapeNetworkSequential(network)
|
||||
: scrapeNetworkParallel(network)),
|
||||
async networkEntity => (networkEntity.parameters?.sequential
|
||||
? scrapeNetworkSequential(networkEntity)
|
||||
: scrapeNetworkParallel(networkEntity)),
|
||||
{ concurrency: 5 },
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user