boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added profile scraping in try/catch block. Re-added boobs enhanced info to actor page.

Browse Source
This commit is contained in:
ThePendulum
2019-11-21 11:43:07 +01:00
parent 31a8f1cac1
commit de36ed97e4
2 changed files with 34 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/actors.js
View File

@@ -183,39 +183,43 @@ function mergeProfiles(profiles, actor) {
async function scrapeActors(actorNames) {
await Promise.map(actorNames || argv.actors, async (actorName) => {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
try {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const profiles = await Promise.all(
Object.values(scrapers.actors)
.map(scraper => scraper.fetchProfile(actorEntry ? actorEntry.name : actorName)),
);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const profiles = await Promise.all(
Object.values(scrapers.actors)
.map(scraper => scraper.fetchProfile(actorEntry ? actorEntry.name : actorName)),
);
const profile = mergeProfiles(profiles, actorEntry);
const profile = mergeProfiles(profiles, actorEntry);
if (profile === null) {
console.log(`Could not find profile for actor '${actorName}'`);
await updateActor(profile, true, false);
if (profile === null) {
console.log(`Could not find profile for actor '${actorName}'`);
await updateActor(profile, true, false);
return;
return;
}
if (actorEntry && profile) {
await createActorMediaDirectory(profile, actorEntry);
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile, actorEntry),
]);
return;
}
const newActorEntry = await storeActor(profile, true, true);
await createActorMediaDirectory(profile, newActorEntry);
await storeAvatars(profile, newActorEntry);
} catch (error) {
console.warn(actorName, error);
}
if (actorEntry && profile) {
await createActorMediaDirectory(profile, actorEntry);
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile, actorEntry),
]);
return;
}
const newActorEntry = await storeActor(profile, true, true);
await createActorMediaDirectory(profile, newActorEntry);
await storeAvatars(profile, newActorEntry);
}, {
concurrency: 3,
});