DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Added Bang! actor scraper. Fixed date interpolation. Showing date and age of death on profile (only if actor has already died).

Browse Source
This commit is contained in:
ThePendulum
2020-05-17 04:59:09 +02:00
parent 7f86399033
commit f42ca7bd52
10 changed files with 183 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/actors.js
View File

@@ -41,7 +41,11 @@ function getMostFrequentDate(dates) {
const month = getMostFrequent(dates.map(dateX => dateX.getMonth()));
const date = getMostFrequent(dates.map(dateX => dateX.getDate()));
return moment({ year, month, date }).toDate();
if (year && month && date) {
return moment({ year, month, date }).toDate();
}
return null;
}
function getLongest(items) {
@@ -201,10 +205,6 @@ async function curateProfile(profile) {
curatedProfile.releases = toBaseReleases(profile.releases);
if (argv.inspect) {
console.log(curatedProfile);
}
return curatedProfile;
} catch (error) {
logger.error(`Failed to curate '${profile.name}': ${error.message}`);
@@ -255,6 +255,8 @@ async function interpolateProfiles(actors) {
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
console.log(valuesByProperty.date_of_birth, profile.date_of_birth);
profile.birth_city = getMostFrequent(valuesByProperty.birth_city);
profile.birth_state = getMostFrequent(valuesByProperty.birth_state);
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.birth_country_alpha2);
@@ -437,10 +439,17 @@ async function scrapeActors(actorNames) {
);
const profiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
const profilesWithAvatarIds = await associateAvatars(profiles);
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actors);
if (argv.inspect) {
console.log(profiles);
}
if (argv.save) {
const profilesWithAvatarIds = await associateAvatars(profiles);
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actors);
}
}
async function getOrCreateActors(baseActors, batchId) {