Added Fame Digital. Added actor release scraping to DDF Network. Improved q and Gamma scraper.

This commit is contained in:
2020-02-06 23:15:28 +01:00
parent db14eaa5f9
commit 6e1de52a40
42 changed files with 752 additions and 168 deletions

View File

@@ -351,6 +351,11 @@ async function scrapeActors(actorNames) {
try {
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
if (!scraper) {
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
}
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
@@ -371,7 +376,7 @@ async function scrapeActors(actorNames) {
} catch (error) {
if (error.warn !== false) {
logger.warn(`Error in scraper ${source}: ${error.message}`);
logger.error(error.stack);
// logger.error(error.stack);
}
}
@@ -380,6 +385,10 @@ async function scrapeActors(actorNames) {
const profile = await mergeProfiles(profiles, actorEntry);
if (argv.inspect) {
console.log(profile);
}
if (profile === null) {
logger.warn(`Could not find profile for actor '${actorName}'`);