forked from DebaucheryLibrarian/traxxx
631 lines
21 KiB
JavaScript
631 lines
21 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const Promise = require('bluebird');
|
|
const moment = require('moment');
|
|
|
|
// const logger = require('./logger')(__filename);
|
|
const knex = require('./knex');
|
|
const scrapers = require('./scrapers/scrapers').actors;
|
|
|
|
const argv = require('./argv');
|
|
const include = require('./utils/argv-include')(argv);
|
|
const logger = require('./logger')(__filename);
|
|
|
|
const { toBaseReleases } = require('./deep');
|
|
const { associateAvatars } = require('./media');
|
|
const { curateSite } = require('./sites');
|
|
|
|
const slugify = require('./utils/slugify');
|
|
const capitalize = require('./utils/capitalize');
|
|
const resolvePlace = require('./utils/resolve-place');
|
|
|
|
const hairColors = {
|
|
'jet-black': 'black',
|
|
'red-head': 'red',
|
|
'soft-black': 'black',
|
|
black: 'black',
|
|
blonde: 'blonde',
|
|
blondie: 'blonde',
|
|
brown: 'brown',
|
|
brunette: 'brown',
|
|
fair: 'blonde',
|
|
raven: 'black',
|
|
red: 'red',
|
|
redhead: 'red',
|
|
};
|
|
|
|
const eyeColors = {
|
|
blue: 'blue',
|
|
brown: 'brown',
|
|
dark: 'brown',
|
|
gray: 'gray',
|
|
green: 'green',
|
|
grey: 'gray',
|
|
hazel: 'hazel',
|
|
};
|
|
|
|
const ethnicities = {
|
|
'african american': 'black',
|
|
'african-american': 'black',
|
|
'native american': 'native american',
|
|
african: 'black',
|
|
aravic: 'arabic',
|
|
asian: 'asian',
|
|
black: 'black',
|
|
caucasian: 'white',
|
|
european: 'white',
|
|
hispanic: 'latina',
|
|
indian: 'indian',
|
|
japanese: 'japanese',
|
|
latina: 'latina',
|
|
latino: 'latino',
|
|
white: 'white',
|
|
};
|
|
|
|
function getMostFrequent(items) {
|
|
const { mostFrequent } = items.reduce((acc, item) => {
|
|
const slug = slugify(item);
|
|
|
|
acc.counts[slug] = (acc.counts[slug] || 0) + 1;
|
|
|
|
if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) {
|
|
acc.mostFrequent = item;
|
|
}
|
|
|
|
return acc;
|
|
}, {
|
|
counts: {},
|
|
mostFrequent: null,
|
|
});
|
|
|
|
return mostFrequent;
|
|
}
|
|
|
|
function getMostFrequentDate(dates) {
|
|
const year = getMostFrequent(dates.map(dateX => dateX.getFullYear()));
|
|
const month = getMostFrequent(dates.map(dateX => dateX.getMonth()));
|
|
const date = getMostFrequent(dates.map(dateX => dateX.getDate()));
|
|
|
|
if (year && month && date) {
|
|
return moment({ year, month, date }).toDate();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function getLongest(items) {
|
|
return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null;
|
|
}
|
|
|
|
function getAverage(items) {
|
|
return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null;
|
|
}
|
|
|
|
function toBaseActors(actorsOrNames, release) {
|
|
return actorsOrNames.map((actorOrName) => {
|
|
const name = capitalize(actorOrName.name || actorOrName);
|
|
const slug = slugify(name);
|
|
|
|
const baseActor = {
|
|
name,
|
|
slug,
|
|
network: release?.site.network,
|
|
};
|
|
|
|
if (actorOrName.name) {
|
|
return {
|
|
...actorOrName,
|
|
...baseActor,
|
|
};
|
|
}
|
|
|
|
return baseActor;
|
|
});
|
|
}
|
|
|
|
function curateActorEntry(baseActor, batchId) {
|
|
return {
|
|
name: baseActor.name,
|
|
slug: baseActor.slug,
|
|
network_id: null,
|
|
batch_id: batchId,
|
|
};
|
|
}
|
|
|
|
function curateActorEntries(baseActors, batchId) {
|
|
return baseActors.map(baseActor => curateActorEntry(baseActor, batchId));
|
|
}
|
|
|
|
function curateProfileEntry(profile) {
|
|
const curatedProfileEntry = {
|
|
actor_id: profile.id,
|
|
site_id: profile.site?.id || null,
|
|
network_id: profile.network?.id || null,
|
|
date_of_birth: profile.dateOfBirth,
|
|
date_of_death: profile.dateOfDeath,
|
|
gender: profile.gender,
|
|
ethnicity: profile.ethnicity,
|
|
description: profile.description,
|
|
birth_city: profile.placeOfBirth?.city || null,
|
|
birth_state: profile.placeOfBirth?.state || null,
|
|
birth_country_alpha2: profile.placeOfBirth?.country || null,
|
|
residence_city: profile.placeOfResidence?.city || null,
|
|
residence_state: profile.placeOfResidence?.state || null,
|
|
residence_country_alpha2: profile.placeOfResidence?.country || null,
|
|
cup: profile.cup,
|
|
bust: profile.bust,
|
|
waist: profile.waist,
|
|
hip: profile.hip,
|
|
natural_boobs: profile.naturalBoobs,
|
|
height: profile.height,
|
|
weight: profile.weight,
|
|
hair: profile.hair,
|
|
eyes: profile.eyes,
|
|
has_tattoos: profile.hasTattoos,
|
|
has_piercings: profile.hasPiercings,
|
|
piercings: profile.piercings,
|
|
tattoos: profile.tattoos,
|
|
avatar_media_id: profile.avatarMediaId || null,
|
|
};
|
|
|
|
return curatedProfileEntry;
|
|
}
|
|
|
|
async function curateProfile(profile) {
|
|
if (!profile) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const curatedProfile = {
|
|
id: profile.id,
|
|
name: profile.name,
|
|
avatar: profile.avatar,
|
|
scraper: profile.scraper,
|
|
site: profile.site,
|
|
network: profile.network,
|
|
};
|
|
|
|
curatedProfile.description = profile.description?.trim() || null;
|
|
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
|
|
|
curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null;
|
|
curatedProfile.hair = hairColors[profile.hair?.trim().toLowerCase()] || null;
|
|
curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null;
|
|
|
|
curatedProfile.tattoos = profile.tattoos?.trim() || null;
|
|
curatedProfile.piercings = profile.piercings?.trim() || null;
|
|
|
|
curatedProfile.gender = (/female/i.test(profile.gender) && 'female')
|
|
|| (/shemale/i.test(profile.gender) && 'transsexual')
|
|
|| (/male/i.test(profile.gender) && 'male')
|
|
|| (/trans/i.test(profile.gender) && 'transsexual')
|
|
|| null;
|
|
|
|
const dateOfBirth = profile.dateOfBirth || profile.birthdate;
|
|
|
|
curatedProfile.dateOfBirth = (!Number.isNaN(Number(dateOfBirth)) // possibly valid date
|
|
&& new Date() - dateOfBirth > 567648000000 // over 18
|
|
&& dateOfBirth)
|
|
|| null;
|
|
|
|
curatedProfile.dateOfDeath = Number.isNaN(Number(profile.dateOfDeath)) ? null : profile.dateOfDeath;
|
|
|
|
curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match(/[a-zA-Z]+/)?.[0]) || null;
|
|
curatedProfile.bust = Number(profile.bust) || profile.bust?.match(/\d+/)?.[0] || null;
|
|
curatedProfile.waist = Number(profile.waist) || profile.waist?.match(/\d+/)?.[0] || null;
|
|
curatedProfile.hip = Number(profile.hip) || profile.hip?.match(/\d+/)?.[0] || null;
|
|
curatedProfile.height = Number(profile.height) || profile.height?.match(/\d+/)?.[0] || null;
|
|
curatedProfile.weight = Number(profile.weight) || profile.weight?.match(/\d+/)?.[0] || null;
|
|
|
|
curatedProfile.naturalBoobs = typeof profile.naturalBoobs === 'boolean' ? profile.naturalBoobs : null;
|
|
curatedProfile.hasTattoos = typeof profile.hasTattoos === 'boolean' ? profile.hasTattoos : null;
|
|
curatedProfile.hasPiercings = typeof profile.hasPiercings === 'boolean' ? profile.hasPiercings : null;
|
|
|
|
if (argv.resolvePlace) {
|
|
const [placeOfBirth, placeOfResidence] = await Promise.all([
|
|
resolvePlace(profile.birthPlace),
|
|
resolvePlace(profile.residencePlace),
|
|
]);
|
|
|
|
curatedProfile.placeOfBirth = placeOfBirth;
|
|
curatedProfile.placeOfResidence = placeOfResidence;
|
|
}
|
|
|
|
if (!curatedProfile.placeOfBirth && curatedProfile.nationality) {
|
|
const country = await knex('countries')
|
|
.where('nationality', 'ilike', `%${curatedProfile.nationality}%`)
|
|
.orderBy('priority', 'desc')
|
|
.first();
|
|
|
|
curatedProfile.placeOfBirth = {
|
|
country: country.alpha2,
|
|
};
|
|
}
|
|
|
|
curatedProfile.social = Array.isArray(profile.social)
|
|
? profile.social.map((social) => {
|
|
try {
|
|
const { href } = new URL(social);
|
|
return href;
|
|
} catch (error) {
|
|
logger.warn(`Profile scraper for '${profile.site.name}' returned invalid social link: ${social}`);
|
|
return null;
|
|
}
|
|
}).filter(Boolean)
|
|
: [];
|
|
|
|
curatedProfile.releases = toBaseReleases(profile.releases);
|
|
|
|
if (profile.ethnicity && !curatedProfile.ethnicity) logger.warn(`Unrecognized ethnicity returned by '${profile.site?.name || profile.network?.slug}' scraper: ${profile.ethnicity}`);
|
|
if (profile.hair && !curatedProfile.hair) logger.warn(`Unrecognized hair color returned by '${profile.site?.name || profile.network?.slug}' scraper: ${profile.hair}`);
|
|
if (profile.eyes && !curatedProfile.eyes) logger.warn(`Unrecognized eye color returned by '${profile.site?.name || profile.network?.slug}' scraper: ${profile.eyes}`);
|
|
|
|
return curatedProfile;
|
|
} catch (error) {
|
|
logger.error(`Failed to curate '${profile.name}': ${error.message}`);
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function interpolateProfiles(actors) {
|
|
const profiles = await knex('actors_profiles')
|
|
.select(['actors_profiles.*', 'media.width as avatar_width', 'media.height as avatar_height', 'media.size as avatar_size'])
|
|
.whereIn('actor_id', actors.map(actor => actor.id))
|
|
.leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id');
|
|
|
|
const profilesByActorId = profiles.reduce((acc, profile) => ({
|
|
...acc,
|
|
[profile.actor_id]: [
|
|
...(acc[profile.actor_id] || []),
|
|
profile,
|
|
],
|
|
}), {});
|
|
|
|
const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => {
|
|
// group values from each profile
|
|
const valuesByProperty = actorProfiles.reduce((acc, profile) => Object
|
|
.entries(profile)
|
|
.reduce((profileAcc, [property, value]) => ({
|
|
...profileAcc,
|
|
[property]: [
|
|
...(acc[property] || []),
|
|
...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value
|
|
],
|
|
}), {
|
|
// bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country
|
|
origin: [...acc.origin || [], {
|
|
...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }),
|
|
...(profile.birth_state && { state: profile.birth_state }),
|
|
...(profile.birth_city && { city: profile.birth_city }),
|
|
}].filter(location => Object.keys(location).length > 0),
|
|
residence: [...acc.residence || [], {
|
|
...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }),
|
|
...(profile.residence_state && { state: profile.residence_state }),
|
|
...(profile.residence_city && { city: profile.residence_city }),
|
|
}].filter(location => Object.keys(location).length > 0),
|
|
}), {});
|
|
|
|
const avatars = actorProfiles.map(profile => profile.avatar_media_id && ({
|
|
id: profile.avatar_media_id,
|
|
width: profile.avatar_width,
|
|
height: profile.avatar_height,
|
|
size: profile.avatar_size,
|
|
})).filter(Boolean);
|
|
|
|
const mostFrequentValues = [
|
|
'gender',
|
|
'ethnicity',
|
|
'cup',
|
|
'bust',
|
|
'waist',
|
|
'hip',
|
|
'natural_boobs',
|
|
'height',
|
|
'hair',
|
|
'eyes',
|
|
'has_tattoos',
|
|
'has_piercings',
|
|
].reduce((acc, property) => ({
|
|
...acc,
|
|
[property]: getMostFrequent(valuesByProperty[property]),
|
|
}), {});
|
|
|
|
const profile = {
|
|
id: actorId,
|
|
...mostFrequentValues,
|
|
};
|
|
|
|
profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth);
|
|
profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death);
|
|
|
|
// ensure most frequent country, city and state match up
|
|
profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map(location => location.country));
|
|
const remainingOriginCountries = valuesByProperty.origin.filter(location => location.country === profile.birth_country_alpha2);
|
|
|
|
profile.birth_state = getMostFrequent(remainingOriginCountries.map(location => location.state));
|
|
const remainingOriginStates = remainingOriginCountries.filter(location => !profile.birth_state || location.state === profile.birth_state);
|
|
|
|
profile.birth_city = getMostFrequent(remainingOriginStates.map(location => location.city));
|
|
|
|
profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map(location => location.country));
|
|
const remainingResidenceCountries = valuesByProperty.residence.filter(location => location.country === profile.residence_country_alpha2);
|
|
|
|
profile.residence_state = getMostFrequent(remainingResidenceCountries.map(location => location.state));
|
|
const remainingResidenceStates = remainingResidenceCountries.filter(location => !profile.residence_state || location.state === profile.residence_state);
|
|
|
|
profile.residence_city = getMostFrequent(remainingResidenceStates.map(location => location.city));
|
|
|
|
profile.weight = getAverage(valuesByProperty.weight);
|
|
|
|
profile.tattoos = getLongest(valuesByProperty.tattoos);
|
|
profile.piercings = getLongest(valuesByProperty.piercings);
|
|
|
|
profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
|
|
|
|
return profile;
|
|
});
|
|
|
|
const transaction = await knex.transaction();
|
|
|
|
const queries = interpolatedProfiles.map(profile => knex('actors')
|
|
.where('id', profile.id)
|
|
.update(profile)
|
|
.transacting(transaction));
|
|
|
|
await Promise.all(queries)
|
|
.then(transaction.commit)
|
|
.catch(transaction.rollback);
|
|
}
|
|
|
|
async function upsertProfiles(profiles) {
|
|
const curatedProfileEntries = profiles.map(profile => curateProfileEntry(profile));
|
|
|
|
const existingProfiles = await knex('actors_profiles')
|
|
.whereIn(['actor_id', 'network_id'], curatedProfileEntries.map(entry => [entry.actor_id, entry.network_id]))
|
|
.orWhereIn(['actor_id', 'site_id'], curatedProfileEntries.map(entry => [entry.actor_id, entry.site_id]));
|
|
|
|
const existingProfilesByActorNetworkSiteIds = existingProfiles.reduce((acc, profile) => ({
|
|
...acc,
|
|
[profile.actor_id]: {
|
|
...acc[profile.actor_id],
|
|
[profile.network_id]: {
|
|
...acc[profile.actor_id]?.[profile.network_id],
|
|
[profile.site_id]: profile,
|
|
},
|
|
},
|
|
}), {});
|
|
|
|
const { updatingProfileEntries, newProfileEntries } = curatedProfileEntries.reduce((acc, profile) => {
|
|
const existingProfile = existingProfilesByActorNetworkSiteIds[profile.actor_id]?.[profile.network_id]?.[profile.site_id];
|
|
|
|
if (existingProfile) {
|
|
return {
|
|
...acc,
|
|
updatingProfileEntries: [...acc.updatingProfileEntries, {
|
|
...profile,
|
|
id: existingProfile.id,
|
|
}],
|
|
};
|
|
}
|
|
|
|
return {
|
|
...acc,
|
|
newProfileEntries: [...acc.newProfileEntries, profile],
|
|
};
|
|
}, {
|
|
updatingProfileEntries: [],
|
|
newProfileEntries: [],
|
|
});
|
|
|
|
if (newProfileEntries.length > 0) {
|
|
await knex('actors_profiles').insert(newProfileEntries);
|
|
}
|
|
|
|
if (argv.force && updatingProfileEntries.length > 0) {
|
|
const transaction = await knex.transaction();
|
|
const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles')
|
|
.where('id', profileEntry.id)
|
|
.update(profileEntry)
|
|
.returning(['id', 'actor_id'])
|
|
.transacting(transaction));
|
|
|
|
await Promise.all(queries)
|
|
.then(transaction.commit)
|
|
.catch(transaction.rollback);
|
|
}
|
|
}
|
|
|
|
async function scrapeProfiles(actor, sources, networksBySlug, sitesBySlug) {
|
|
const profiles = Promise.map(sources, async (source) => {
|
|
try {
|
|
// config may group sources to try until success
|
|
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
|
try {
|
|
const scraper = scrapers[scraperSlug];
|
|
const context = {
|
|
site: sitesBySlug[scraperSlug] || null,
|
|
network: networksBySlug[scraperSlug] || sitesBySlug[scraperSlug]?.network || null,
|
|
scraper: scraperSlug,
|
|
};
|
|
|
|
if (!scraper?.fetchProfile) {
|
|
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
|
|
throw new Error(`No profile profile scraper available for ${scraperSlug}`);
|
|
}
|
|
|
|
if (!context.site && !context.network) {
|
|
logger.warn(`No site or network found for ${scraperSlug}`);
|
|
throw new Error(`No site or network found for ${scraperSlug}`);
|
|
}
|
|
|
|
logger.verbose(`Searching profile for '${actor.name}' on '${scraperSlug}'`);
|
|
|
|
const profile = await scraper.fetchProfile(actor.name, context, include);
|
|
|
|
if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure
|
|
logger.verbose(`Profile for '${actor.name}' not available on ${context.site?.name || context.network?.name || context.scraper}, scraper returned ${profile}`);
|
|
throw Object.assign(new Error(`Profile for '${actor.name}' not available on ${context.site?.name || context.network?.name || context.scraper}`), { code: 'PROFILE_NOT_AVAILABLE' });
|
|
}
|
|
|
|
return {
|
|
...actor,
|
|
...profile,
|
|
...context,
|
|
};
|
|
} catch (error) {
|
|
if (error.code !== 'PROFILE_NOT_AVAILABLE') {
|
|
logger.error(`Failed to fetch profile for '${actor.name}' from '${scraperSlug}': ${error.message}`);
|
|
}
|
|
|
|
throw error;
|
|
}
|
|
}), Promise.reject(new Error()));
|
|
} catch (error) {
|
|
if (error.code !== 'PROFILE_NOT_AVAILABLE') {
|
|
logger.error(`Failed to fetch profile for '${actor.name}': ${error.message}`);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
});
|
|
|
|
return profiles.filter(Boolean);
|
|
}
|
|
|
|
async function scrapeActors(actorNames) {
|
|
const baseActors = toBaseActors(actorNames);
|
|
|
|
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
|
|
const siteSlugs = sources.flat();
|
|
|
|
const [networks, sites, existingActorEntries] = await Promise.all([
|
|
knex('networks').whereIn('slug', siteSlugs),
|
|
knex('sites')
|
|
.select(
|
|
'sites.*',
|
|
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
|
|
)
|
|
.whereIn('sites.slug', siteSlugs)
|
|
.leftJoin('networks', 'sites.network_id', 'networks.id'),
|
|
knex('actors')
|
|
.select(['id', 'name', 'slug'])
|
|
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
|
|
.whereNull('network_id'),
|
|
]);
|
|
|
|
const existingActorEntriesBySlug = existingActorEntries.reduce((acc, actorEntry) => ({ ...acc, [actorEntry.slug]: actorEntry }), {});
|
|
|
|
const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: network }), {});
|
|
const sitesBySlug = sites.reduce((acc, site) => ({ ...acc, [site.slug]: curateSite(site) }), {});
|
|
|
|
const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlug[baseActor.slug]);
|
|
|
|
const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null];
|
|
const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId);
|
|
const newActorEntries = batchId && await knex('actors').insert(curatedActorEntries).returning(['id', 'name', 'slug']);
|
|
|
|
const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []);
|
|
|
|
// TODO: don't fetch existing profiles unless --force is used
|
|
|
|
const profilesPerActor = await Promise.map(
|
|
actors,
|
|
async actor => scrapeProfiles(actor, sources, networksBySlug, sitesBySlug),
|
|
{ concurrency: 10 },
|
|
);
|
|
|
|
const curatedProfiles = await Promise.all(profilesPerActor.flat().map(profile => curateProfile(profile)));
|
|
const profiles = curatedProfiles.filter(Boolean);
|
|
|
|
if (argv.inspect) {
|
|
console.log(profiles);
|
|
}
|
|
|
|
if (argv.save) {
|
|
const profilesWithAvatarIds = await associateAvatars(profiles);
|
|
|
|
await upsertProfiles(profilesWithAvatarIds);
|
|
await interpolateProfiles(actors);
|
|
}
|
|
|
|
return profiles;
|
|
}
|
|
|
|
async function getOrCreateActors(baseActors, batchId) {
|
|
const existingActors = await knex('actors')
|
|
.select('id', 'alias_for', 'name', 'slug', 'network_id')
|
|
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
|
|
.whereNull('network_id')
|
|
.orWhereIn(['slug', 'network_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.network.id]));
|
|
|
|
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
|
|
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
|
|
...acc,
|
|
[actor.network_id]: {
|
|
...acc[actor.network_id],
|
|
[actor.slug]: true,
|
|
},
|
|
}), {});
|
|
|
|
const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.network.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]);
|
|
|
|
const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId);
|
|
const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'network_id']);
|
|
|
|
if (Array.isArray(newActors)) {
|
|
return newActors.concat(existingActors);
|
|
}
|
|
|
|
return existingActors;
|
|
}
|
|
|
|
async function associateActors(releases, batchId) {
|
|
const baseActorsByReleaseId = releases.reduce((acc, release) => {
|
|
if (release.actors) {
|
|
acc[release.id] = toBaseActors(release.actors, release);
|
|
}
|
|
|
|
return acc;
|
|
}, {});
|
|
|
|
const baseActors = Object.values(baseActorsByReleaseId).flat();
|
|
|
|
if (baseActors.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const baseActorsBySlug = baseActors.reduce((acc, baseActor) => ({
|
|
...acc,
|
|
[baseActor.slug]: baseActor,
|
|
}), {});
|
|
|
|
const uniqueBaseActors = Object.values(baseActorsBySlug);
|
|
|
|
const actors = await getOrCreateActors(uniqueBaseActors, batchId);
|
|
|
|
const actorIdsBySlug = actors.reduce((acc, actor) => ({
|
|
...acc,
|
|
[actor.slug]: actor.alias_for || actor.id,
|
|
}), {});
|
|
|
|
const releaseActorAssociations = Object.entries(baseActorsByReleaseId)
|
|
.map(([releaseId, releaseActors]) => releaseActors
|
|
.map(releaseActor => ({
|
|
release_id: releaseId,
|
|
actor_id: actorIdsBySlug[releaseActor.slug],
|
|
})))
|
|
.flat();
|
|
|
|
await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`);
|
|
|
|
return actors;
|
|
}
|
|
|
|
module.exports = {
|
|
associateActors,
|
|
scrapeActors,
|
|
};
|