DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Resolving actor birth and residence place before storage. Layout improvements.

Browse Source
This commit is contained in:
ThePendulum
2019-11-29 05:46:06 +01:00
parent 4be508b388
commit 0dbe853f39
17 changed files with 377 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
src/actors.js
View File

@@ -6,6 +6,7 @@ const knex = require('./knex');
const argv = require('./argv');
const scrapers = require('./scrapers/scrapers');
const whereOr = require('./utils/where-or');
const resolvePlace = require('./utils/resolve-place');
const { createActorMediaDirectory, storeAvatars } = require('./media');
async function curateActor(actor) {
@@ -18,27 +19,15 @@ async function curateActor(actor) {
.where({ domain: 'actors', target_id: actor.id }),
]);
return {
const curatedActor = {
id: actor.id,
gender: actor.gender,
name: actor.name,
description: actor.description,
birthdate: actor.birthdate && new Date(actor.birthdate),
country: actor.country_alpha2,
residencePlace: actor.residence_place,
residenceCountry: actor.residence_country_alpha2
? {
alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name,
}
: null,
birthPlace: actor.birth_place,
birthCountry: actor.birth_country_alpha2
? {
alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name,
}
: null,
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
ethnicity: actor.ethnicity,
height: actor.height,
weight: actor.weight,
@@ -50,9 +39,35 @@ async function curateActor(actor) {
slug: actor.slug,
avatar: photos.find(photo => photo.role === 'avatar'),
photos: photos.filter(photo => photo.role === 'photo'),
hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
social,
scrapedAt: actor.scraped_at,
};
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
if (actor.birth_country_alpha2) {
curatedActor.origin.country = {
alpha2: actor.birth_country_alpha2,
name: actor.birth_country_name,
};
}
if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
if (actor.residence_country_alpha2) {
curatedActor.residence.country = {
alpha2: actor.residence_country_alpha2,
name: actor.residence_country_name,
};
}
return curatedActor;
}
function curateActors(releases) {
@@ -70,10 +85,6 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
description: actor.description,
gender: actor.gender,
ethnicity: actor.ethnicity,
birth_country_alpha2: actor.birthCountry,
residence_country_alpha2: actor.residenceCountry,
birth_place: actor.birthPlace,
residence_place: actor.residencePlace,
bust: actor.bust,
waist: actor.waist,
hip: actor.hip,
@@ -92,6 +103,18 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
curatedActor.id = actor.id;
}
if (actor.birthPlace) {
curatedActor.birth_city = actor.birthPlace.city;
curatedActor.birth_state = actor.birthPlace.state;
curatedActor.birth_country_alpha2 = actor.birthPlace.country;
}
if (actor.residencePlace) {
curatedActor.residence_city = actor.residencePlace.city;
curatedActor.residence_state = actor.residencePlace.state;
curatedActor.residence_country_alpha2 = actor.residencePlace.country;
}
if (scraped) {
curatedActor.scraped_at = new Date();
curatedActor.scrape_success = scrapeSuccess;
@@ -102,7 +125,7 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
function curateSocialEntry(url, actorId) {
const { hostname, origin, pathname } = new URL(url);
const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube'].find(platformName => hostname.match(platformName));
const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube', 'fancentro'].find(platformName => hostname.match(platformName));
return {
url: `${origin}${pathname}`,
@@ -184,8 +207,8 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
return actorEntry;
}
function mergeProfiles(profiles, actor) {
return profiles.reduce((prevProfile, profile) => {
async function mergeProfiles(profiles, actor) {
const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) {
return prevProfile;
}
@@ -196,21 +219,19 @@ function mergeProfiles(profiles, actor) {
description: prevProfile.description || profile.description,
gender: prevProfile.gender || profile.gender,
birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
birthCountry: prevProfile.birthCountry || profile.birthCountry,
residenceCountry: prevProfile.residenceCountry || profile.residenceCountry,
birthPlace: prevProfile.birthPlace || profile.birthPlace,
residencePlace: prevProfile.residencePlace || profile.residencePlace,
ethnicity: prevProfile.ethnicity || profile.ethnicity,
bust: prevProfile.bust || profile.bust,
waist: prevProfile.waist || profile.waist,
hip: prevProfile.hip || profile.hip,
naturalBoobs: prevProfile.naturalBoobs || profile.naturalBoobs,
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
height: prevProfile.height || profile.height,
weight: prevProfile.weight || profile.weight,
hair: prevProfile.hair || profile.hair,
eyes: prevProfile.eyes || profile.eyes,
hasPiercings: prevProfile.hasPiercings || profile.hasPiercings,
hasTattoos: prevProfile.hasTattoos || profile.hasTattoos,
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
piercings: prevProfile.piercings || profile.piercings,
tattoos: prevProfile.tattoos || profile.tattoos,
social: prevProfile.social.concat(profile.social || []),
@@ -220,6 +241,16 @@ function mergeProfiles(profiles, actor) {
social: [],
avatars: [],
});
const [birthPlace, residencePlace] = await Promise.all([
resolvePlace(mergedProfile.birthPlace),
resolvePlace(mergedProfile.residencePlace),
]);
mergedProfile.birthPlace = birthPlace;
mergedProfile.residencePlace = residencePlace;
return mergedProfile;
}
async function scrapeActors(actorNames) {
@@ -228,35 +259,44 @@ async function scrapeActors(actorNames) {
const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const profiles = await Promise.all(
Object.values(scrapers.actors)
.map(scraper => scraper.fetchProfile(actorEntry ? actorEntry.name : actorName)),
);
const profiles = await Promise.map(Object.entries(scrapers.actors), async ([scraperSlug, scraper]) => {
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName);
const profile = mergeProfiles(profiles, actorEntry);
return {
scraper: scraperSlug,
...profile,
};
});
const profile = await mergeProfiles(profiles, actorEntry);
if (profile === null) {
console.log(`Could not find profile for actor '${actorName}'`);
await updateActor(profile, true, false);
if (argv.save) {
await updateActor(profile, true, false);
}
return;
}
if (actorEntry && profile) {
await createActorMediaDirectory(profile, actorEntry);
if (argv.save) {
if (actorEntry && profile) {
await createActorMediaDirectory(profile, actorEntry);
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile, actorEntry),
]);
await Promise.all([
updateActor(profile, true, true),
storeAvatars(profile, actorEntry),
]);
return;
return;
}
const newActorEntry = await storeActor(profile, true, true);
await createActorMediaDirectory(profile, newActorEntry);
await storeAvatars(profile, newActorEntry);
}
const newActorEntry = await storeActor(profile, true, true);
await createActorMediaDirectory(profile, newActorEntry);
await storeAvatars(profile, newActorEntry);
} catch (error) {
console.warn(actorName, error);
}