Added option to fetch all of an actor's releases (for supporter scrapers), and a utility to extract posters.

This commit is contained in:
2020-01-31 01:55:55 +01:00
parent 4ecb386233
commit ffdf0690e7
5 changed files with 58 additions and 5 deletions

View File

@@ -303,10 +303,12 @@ async function mergeProfiles(profiles, actor) {
tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []),
avatars: prevProfile.avatars.concat(profile.avatar ? [profile.avatar] : []), // don't flatten fallbacks
releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
};
}, {
social: [],
avatars: [],
releases: [],
});
const [birthPlace, residencePlace] = await Promise.all([
@@ -332,7 +334,7 @@ async function mergeProfiles(profiles, actor) {
}
async function scrapeActors(actorNames) {
await Promise.map(actorNames || argv.actors, async (actorName) => {
return Promise.map(actorNames || argv.actors, async (actorName) => {
try {
const actorSlug = slugify(actorName);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
@@ -362,7 +364,7 @@ async function scrapeActors(actorNames) {
await updateActor(profile, true, false);
}
return;
return null;
}
if (argv.save) {
@@ -382,7 +384,7 @@ async function scrapeActors(actorNames) {
}, actorEntry.name),
]);
return;
return profile;
}
const newActorEntry = await storeActor(profile, true, true);
@@ -397,8 +399,12 @@ async function scrapeActors(actorNames) {
naming: 'timestamp',
}, newActorEntry.name);
}
return profile;
} catch (error) {
logger.warn(`${actorName}: ${error}`);
return null;
}
}, {
concurrency: 3,