DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Fixed actor birthdate and social links scraping.

This commit is contained in:
ThePendulum 2019-11-27 05:12:51 +01:00
parent d113123778
commit 884ef248e4
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/actors.js
View File

@ -141,6 +141,17 @@ async function fetchActors(queryObject) {
return curateActors(releases); return curateActors(releases);
} }
async function storeSocialLinks(actor) {
const existingSocialLinks = await knex('social').where({
domain: 'actors',
target_id: actor.id,
});
const newSocialLinks = actor.social.filter(url => !existingSocialLinks.some(existingLink => url === existingLink.url));
await knex('social').insert(curateSocialEntries(newSocialLinks, actor));
}
async function storeActor(actor, scraped = false, scrapeSuccess = false) { async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess); const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
@ -148,7 +159,7 @@ async function storeActor(actor, scraped = false, scrapeSuccess = false) {
.insert(curatedActor) .insert(curatedActor)
.returning('*'); .returning('*');
await knex('social').insert(curateSocialEntries(actor.social, actor)); await storeSocialLinks({ ...actor, ...actorEntry });
console.log(`Added new entry for actor '${actor.name}'`); console.log(`Added new entry for actor '${actor.name}'`);
@ -163,7 +174,7 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
.update(curatedActor) .update(curatedActor)
.returning('*'); .returning('*');
await knex('social').insert(curateSocialEntries(actor.social, actor)); await storeSocialLinks({ ...actor, ...curatedActor, ...actorEntry });
console.log(`Updated entry for actor '${actor.name}'`); console.log(`Updated entry for actor '${actor.name}'`);
@ -181,7 +192,7 @@ function mergeProfiles(profiles, actor) {
name: actor ? actor.name : profile.name, name: actor ? actor.name : profile.name,
description: prevProfile.description || profile.description, description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender, gender: prevProfile.gender || profile.gender,
birthdate: Number.isNaN(prevProfile.birthdate) ? profile.birthdate : prevProfile.birthdate, birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
birthCountry: prevProfile.birthCountry || profile.birthCountry, birthCountry: prevProfile.birthCountry || profile.birthCountry,
residenceCountry: prevProfile.residenceCountry || profile.residenceCountry, residenceCountry: prevProfile.residenceCountry || profile.residenceCountry,
birthPlace: prevProfile.birthPlace || profile.birthPlace, birthPlace: prevProfile.birthPlace || profile.birthPlace,