2019-11-10 03:20:22 +00:00
'use strict' ;
2020-05-14 02:26:05 +00:00
const config = require ( 'config' ) ;
const Promise = require ( 'bluebird' ) ;
2020-05-17 01:00:44 +00:00
const moment = require ( 'moment' ) ;
2020-05-14 02:26:05 +00:00
2020-05-13 00:56:20 +00:00
// const logger = require('./logger')(__filename);
2020-03-26 02:32:07 +00:00
const knex = require ( './knex' ) ;
2020-05-15 02:40:59 +00:00
const scrapers = require ( './scrapers/scrapers' ) . actors ;
2020-05-14 02:26:05 +00:00
const argv = require ( './argv' ) ;
2020-05-15 02:40:59 +00:00
const include = require ( './utils/argv-include' ) ( argv ) ;
const logger = require ( './logger' ) ( _ _filename ) ;
2020-05-17 01:00:44 +00:00
const { toBaseReleases } = require ( './deep' ) ;
const { associateAvatars } = require ( './media' ) ;
2020-05-18 01:34:41 +00:00
const { curateSite } = require ( './sites' ) ;
2020-05-17 01:00:44 +00:00
2020-01-07 03:23:28 +00:00
const slugify = require ( './utils/slugify' ) ;
2020-03-26 02:32:07 +00:00
const capitalize = require ( './utils/capitalize' ) ;
2020-05-15 02:40:59 +00:00
const resolvePlace = require ( './utils/resolve-place' ) ;
2020-05-17 01:00:44 +00:00
function getMostFrequent ( items ) {
const { mostFrequent } = items . reduce ( ( acc , item ) => {
acc . counts [ item ] = ( acc . counts [ item ] || 0 ) + 1 ;
if ( ! acc . mostFrequent || acc . counts [ item ] > acc . counts [ acc . mostFrequent ] ) {
acc . mostFrequent = item ;
}
return acc ;
} , {
counts : { } ,
mostFrequent : null ,
} ) ;
return mostFrequent ;
}
function getMostFrequentDate ( dates ) {
const year = getMostFrequent ( dates . map ( dateX => dateX . getFullYear ( ) ) ) ;
const month = getMostFrequent ( dates . map ( dateX => dateX . getMonth ( ) ) ) ;
const date = getMostFrequent ( dates . map ( dateX => dateX . getDate ( ) ) ) ;
2020-05-17 02:59:09 +00:00
if ( year && month && date ) {
return moment ( { year , month , date } ) . toDate ( ) ;
}
return null ;
2020-05-17 01:00:44 +00:00
}
function getLongest ( items ) {
return items . sort ( ( itemA , itemB ) => itemB . length - itemA . length ) [ 0 ] || null ;
}
function getAverage ( items ) {
2020-05-17 01:04:58 +00:00
return Math . round ( items . reduce ( ( acc , item ) => acc + item , 0 ) / items . length ) || null ;
2020-05-17 01:00:44 +00:00
}
2019-11-10 03:20:22 +00:00
2020-03-26 02:32:07 +00:00
function toBaseActors ( actorsOrNames , release ) {
2020-05-14 02:26:05 +00:00
return actorsOrNames . map ( ( actorOrName ) => {
const name = capitalize ( actorOrName . name || actorOrName ) ;
const slug = slugify ( name ) ;
const baseActor = {
name ,
slug ,
network : release ? . site . network ,
} ;
if ( actorOrName . name ) {
return {
... actorOrName ,
... baseActor ,
} ;
}
return baseActor ;
} ) ;
2020-03-24 02:48:24 +00:00
}
2020-05-13 00:56:20 +00:00
function curateActorEntry ( baseActor , batchId ) {
2020-05-14 02:26:05 +00:00
return {
name : baseActor . name ,
slug : baseActor . slug ,
network _id : null ,
batch _id : batchId ,
} ;
2020-03-26 02:32:07 +00:00
}
2020-05-13 00:56:20 +00:00
function curateActorEntries ( baseActors , batchId ) {
2020-05-14 02:26:05 +00:00
return baseActors . map ( baseActor => curateActorEntry ( baseActor , batchId ) ) ;
2020-03-26 02:32:07 +00:00
}
2020-05-15 02:40:59 +00:00
function curateProfileEntry ( profile ) {
const curatedProfileEntry = {
actor _id : profile . id ,
site _id : profile . site ? . id || null ,
network _id : profile . network ? . id || null ,
date _of _birth : profile . dateOfBirth ,
date _of _death : profile . dateOfDeath ,
gender : profile . gender ,
ethnicity : profile . ethnicity ,
description : profile . description ,
birth _city : profile . placeOfBirth ? . city || null ,
birth _state : profile . placeOfBirth ? . state || null ,
2020-05-17 01:00:44 +00:00
birth _country _alpha2 : profile . placeOfBirth ? . country || null ,
2020-05-15 02:40:59 +00:00
residence _city : profile . placeOfResidence ? . city || null ,
residence _state : profile . placeOfResidence ? . state || null ,
2020-05-17 01:00:44 +00:00
residence _country _alpha2 : profile . placeOfResidence ? . country || null ,
2020-05-15 02:40:59 +00:00
cup : profile . cup ,
bust : profile . bust ,
waist : profile . waist ,
hip : profile . hip ,
natural _boobs : profile . naturalBoobs ,
height : profile . height ,
weight : profile . weight ,
hair : profile . hair ,
eyes : profile . eyes ,
has _tattoos : profile . hasTattoos ,
has _piercings : profile . hasPiercings ,
piercings : profile . piercings ,
tattoos : profile . tattoos ,
2020-05-16 02:36:45 +00:00
avatar _media _id : profile . avatarMediaId || null ,
2020-05-15 02:40:59 +00:00
} ;
return curatedProfileEntry ;
}
async function curateProfile ( profile ) {
2020-05-17 23:22:56 +00:00
if ( ! profile ) {
return null ;
}
2020-05-15 02:40:59 +00:00
try {
const curatedProfile = {
id : profile . id ,
name : profile . name ,
avatar : profile . avatar ,
2020-05-16 02:36:45 +00:00
scraper : profile . scraper ,
2020-05-18 01:22:03 +00:00
site : profile . site ,
network : profile . network ,
2020-05-15 02:40:59 +00:00
} ;
curatedProfile . description = profile . description ? . trim ( ) || null ;
curatedProfile . nationality = profile . nationality ? . trim ( ) || null ; // used to derive country when country not available
curatedProfile . ethnicity = profile . ethnicity ? . trim ( ) || null ;
curatedProfile . hair = profile . hair ? . trim ( ) || null ;
curatedProfile . eyes = profile . eyes ? . trim ( ) || null ;
curatedProfile . tattoos = profile . tattoos ? . trim ( ) || null ;
curatedProfile . piercings = profile . piercings ? . trim ( ) || null ;
curatedProfile . gender = ( /female/i . test ( profile . gender ) && 'female' )
|| ( /shemale/i . test ( profile . gender ) && 'transsexual' )
|| ( /male/i . test ( profile . gender ) && 'male' )
|| ( /trans/i . test ( profile . gender ) && 'transsexual' )
|| null ;
2020-05-17 03:08:41 +00:00
const dateOfBirth = profile . dateOfBirth || profile . birthdate ;
curatedProfile . dateOfBirth = ( ! Number . isNaN ( Number ( dateOfBirth ) ) // possibly valid date
&& new Date ( ) - dateOfBirth > 567648000000 // over 18
&& dateOfBirth )
2020-05-15 02:40:59 +00:00
|| null ;
curatedProfile . dateOfDeath = Number . isNaN ( Number ( profile . dateOfDeath ) ) ? null : profile . dateOfDeath ;
2020-05-17 23:22:56 +00:00
curatedProfile . cup = profile . cup || ( typeof profile . bust === 'string' && profile . bust ? . match ( /[a-zA-Z]+/ ) ? . [ 0 ] ) || null ;
2020-05-15 02:40:59 +00:00
curatedProfile . bust = Number ( profile . bust ) || profile . bust ? . match ( /\d+/ ) ? . [ 0 ] || null ;
curatedProfile . waist = Number ( profile . waist ) || profile . waist ? . match ( /\d+/ ) ? . [ 0 ] || null ;
curatedProfile . hip = Number ( profile . hip ) || profile . hip ? . match ( /\d+/ ) ? . [ 0 ] || null ;
curatedProfile . height = Number ( profile . height ) || profile . height ? . match ( /\d+/ ) ? . [ 0 ] || null ;
curatedProfile . weight = Number ( profile . weight ) || profile . weight ? . match ( /\d+/ ) ? . [ 0 ] || null ;
curatedProfile . naturalBoobs = typeof profile . naturalBoobs === 'boolean' ? profile . naturalBoobs : null ;
curatedProfile . hasTattoos = typeof profile . hasTattoos === 'boolean' ? profile . hasTattoos : null ;
curatedProfile . hasPiercings = typeof profile . hasPiercings === 'boolean' ? profile . hasPiercings : null ;
2020-05-17 01:00:44 +00:00
if ( argv . resolvePlace ) {
const [ placeOfBirth , placeOfResidence ] = await Promise . all ( [
resolvePlace ( profile . birthPlace ) ,
resolvePlace ( profile . residencePlace ) ,
] ) ;
2020-05-15 02:40:59 +00:00
2020-05-17 01:00:44 +00:00
curatedProfile . placeOfBirth = placeOfBirth ;
curatedProfile . placeOfResidence = placeOfResidence ;
}
2020-05-15 02:40:59 +00:00
if ( ! curatedProfile . placeOfBirth && curatedProfile . nationality ) {
const country = await knex ( 'countries' )
. where ( 'nationality' , 'ilike' , ` % ${ curatedProfile . nationality } % ` )
. orderBy ( 'priority' , 'desc' )
. first ( ) ;
curatedProfile . placeOfBirth = {
country : country . alpha2 ,
} ;
}
curatedProfile . social = Array . isArray ( profile . social )
? profile . social . map ( ( social ) => {
try {
2020-05-17 03:24:46 +00:00
const { href } = new URL ( social ) ;
2020-05-15 02:40:59 +00:00
return href ;
} catch ( error ) {
logger . warn ( ` Profile scraper for ' ${ profile . site . name } ' returned invalid social link: ${ social } ` ) ;
return null ;
}
} ) . filter ( Boolean )
: [ ] ;
curatedProfile . releases = toBaseReleases ( profile . releases ) ;
return curatedProfile ;
} catch ( error ) {
logger . error ( ` Failed to curate ' ${ profile . name } ': ${ error . message } ` ) ;
return null ;
}
}
2020-05-17 01:00:44 +00:00
async function interpolateProfiles ( actors ) {
const profiles = await knex ( 'actors_profiles' )
. select ( [ 'actors_profiles.*' , 'media.width as avatar_width' , 'media.height as avatar_height' , 'media.size as avatar_size' ] )
. whereIn ( 'actor_id' , actors . map ( actor => actor . id ) )
. leftJoin ( 'media' , 'actors_profiles.avatar_media_id' , 'media.id' ) ;
const profilesByActorId = profiles . reduce ( ( acc , profile ) => ( {
... acc ,
[ profile . actor _id ] : [
... ( acc [ profile . actor _id ] || [ ] ) ,
profile ,
] ,
} ) , { } ) ;
const interpolatedProfiles = Object . entries ( profilesByActorId ) . map ( ( [ actorId , actorProfiles ] ) => {
const valuesByProperty = actorProfiles . reduce ( ( acc , profile ) => Object
. entries ( profile )
. reduce ( ( profileAcc , [ property , value ] ) => ( {
... profileAcc ,
[ property ] : [
... ( acc [ property ] || [ ] ) ,
... ( value === null ? [ ] : [ value ] ) ,
] ,
} ) , { } ) , { } ) ;
const avatars = actorProfiles . map ( profile => profile . avatar _media _id && ( {
id : profile . avatar _media _id ,
width : profile . avatar _width ,
height : profile . avatar _height ,
size : profile . avatar _size ,
} ) ) . filter ( Boolean ) ;
const profile = {
id : actorId ,
} ;
profile . gender = getMostFrequent ( valuesByProperty . gender ) ;
profile . ethnicity = getMostFrequent ( valuesByProperty . ethnicity . map ( ethnicity => ethnicity . toLowerCase ( ) ) ) ;
profile . date _of _birth = getMostFrequentDate ( valuesByProperty . date _of _birth ) ;
profile . date _of _death = getMostFrequentDate ( valuesByProperty . date _of _death ) ;
2020-05-17 23:22:56 +00:00
// TODO: fix city, state and country not matching
2020-05-17 01:00:44 +00:00
profile . birth _city = getMostFrequent ( valuesByProperty . birth _city ) ;
profile . birth _state = getMostFrequent ( valuesByProperty . birth _state ) ;
profile . birth _country _alpha2 = getMostFrequent ( valuesByProperty . birth _country _alpha2 ) ;
profile . residence _city = getMostFrequent ( valuesByProperty . residence _city ) ;
profile . residence _state = getMostFrequent ( valuesByProperty . residence _state ) ;
profile . residence _country _alpha2 = getMostFrequent ( valuesByProperty . residence _country _alpha2 ) ;
profile . cup = getMostFrequent ( valuesByProperty . cup ) ;
profile . bust = getMostFrequent ( valuesByProperty . bust ) ;
profile . waist = getMostFrequent ( valuesByProperty . waist ) ;
profile . hip = getMostFrequent ( valuesByProperty . hip ) ;
profile . natural _boobs = getMostFrequent ( valuesByProperty . natural _boobs ) ;
profile . hair = getMostFrequent ( valuesByProperty . hair . map ( hair => hair . toLowerCase ( ) ) ) ;
profile . eyes = getMostFrequent ( valuesByProperty . eyes . map ( eyes => eyes . toLowerCase ( ) ) ) ;
profile . weight = getAverage ( valuesByProperty . weight ) ;
profile . height = getMostFrequent ( valuesByProperty . height ) ;
profile . has _tattoos = getMostFrequent ( valuesByProperty . has _tattoos ) ;
profile . has _piercings = getMostFrequent ( valuesByProperty . has _piercings ) ;
profile . tattoos = getLongest ( valuesByProperty . tattoos ) ;
profile . piercings = getLongest ( valuesByProperty . piercings ) ;
2020-05-18 01:22:03 +00:00
profile . avatar _media _id = avatars . sort ( ( avatarA , avatarB ) => avatarB . height - avatarA . height ) [ 0 ] ? . id || null ;
2020-05-17 01:00:44 +00:00
return profile ;
} ) ;
const transaction = await knex . transaction ( ) ;
const queries = interpolatedProfiles . map ( profile => knex ( 'actors' )
. where ( 'id' , profile . id )
. update ( profile )
. transacting ( transaction ) ) ;
await Promise . all ( queries )
. then ( transaction . commit )
. catch ( transaction . rollback ) ;
}
async function upsertProfiles ( profiles ) {
const curatedProfileEntries = profiles . map ( profile => curateProfileEntry ( profile ) ) ;
2020-05-15 02:40:59 +00:00
const existingProfiles = await knex ( 'actors_profiles' )
. whereIn ( [ 'actor_id' , 'network_id' ] , curatedProfileEntries . map ( entry => [ entry . actor _id , entry . network _id ] ) )
. orWhereIn ( [ 'actor_id' , 'site_id' ] , curatedProfileEntries . map ( entry => [ entry . actor _id , entry . site _id ] ) ) ;
const existingProfilesByActorNetworkSiteIds = existingProfiles . reduce ( ( acc , profile ) => ( {
... acc ,
[ profile . actor _id ] : {
... acc [ profile . actor _id ] ,
[ profile . network _id ] : {
... acc [ profile . actor _id ] ? . [ profile . network _id ] ,
[ profile . site _id ] : profile ,
} ,
} ,
} ) , { } ) ;
const { updatingProfileEntries , newProfileEntries } = curatedProfileEntries . reduce ( ( acc , profile ) => {
const existingProfile = existingProfilesByActorNetworkSiteIds [ profile . actor _id ] ? . [ profile . network _id ] ? . [ profile . site _id ] ;
if ( existingProfile ) {
return {
... acc ,
updatingProfileEntries : [ ... acc . updatingProfileEntries , {
... profile ,
id : existingProfile . id ,
} ] ,
} ;
}
return {
... acc ,
newProfileEntries : [ ... acc . newProfileEntries , profile ] ,
} ;
} , {
updatingProfileEntries : [ ] ,
newProfileEntries : [ ] ,
} ) ;
if ( newProfileEntries . length > 0 ) {
await knex ( 'actors_profiles' ) . insert ( newProfileEntries ) ;
}
if ( argv . force && updatingProfileEntries . length > 0 ) {
2020-05-16 02:36:45 +00:00
const transaction = await knex . transaction ( ) ;
const queries = updatingProfileEntries . map ( profileEntry => knex ( 'actors_profiles' )
. where ( 'id' , profileEntry . id )
. update ( profileEntry )
. returning ( [ 'id' , 'actor_id' ] )
. transacting ( transaction ) ) ;
await Promise . all ( queries )
. then ( transaction . commit )
. catch ( transaction . rollback ) ;
2020-05-15 02:40:59 +00:00
}
}
2020-05-17 23:22:56 +00:00
async function scrapeProfiles ( actor , sources , networksBySlug , sitesBySlug ) {
const profiles = Promise . map ( sources , async ( source ) => {
try {
return await [ ] . concat ( source ) . reduce ( async ( outcome , scraperSlug ) => outcome . catch ( async ( ) => {
const scraper = scrapers [ scraperSlug ] ;
2020-05-18 01:22:03 +00:00
const context = {
site : sitesBySlug [ scraperSlug ] || null ,
2020-05-18 01:34:41 +00:00
network : networksBySlug [ scraperSlug ] || sitesBySlug [ scraperSlug ] ? . network || null ,
2020-05-18 01:22:03 +00:00
scraper : scraperSlug ,
} ;
2020-05-17 23:22:56 +00:00
if ( ! scraper ? . fetchProfile ) {
logger . warn ( ` No profile profile scraper available for ${ scraperSlug } ` ) ;
throw new Error ( ` No profile profile scraper available for ${ scraperSlug } ` ) ;
}
2020-05-18 01:22:03 +00:00
if ( ! context . site && ! context . network ) {
2020-05-17 23:22:56 +00:00
logger . warn ( ` No site or network found for ${ scraperSlug } ` ) ;
throw new Error ( ` No site or network found for ${ scraperSlug } ` ) ;
}
logger . verbose ( ` Searching profile for ' ${ actor . name } ' on ' ${ scraperSlug } ' ` ) ;
2020-05-18 01:22:03 +00:00
const profile = await scraper . fetchProfile ( actor . name , context , include ) ;
2020-05-17 23:22:56 +00:00
if ( ! profile || typeof profile === 'number' ) { // scraper returns HTTP code on request failure
logger . verbose ( ` Profile for ' ${ actor . name } ' not available on ${ scraperSlug } , scraper returned ${ profile } ` ) ;
throw Object . assign ( new Error ( ` Profile for ' ${ actor . name } ' not available on ${ scraperSlug } ` ) , { code : 'PROFILE_NOT_AVAILABLE' } ) ;
}
return {
... actor ,
... profile ,
2020-05-18 01:22:03 +00:00
... context ,
2020-05-17 23:22:56 +00:00
} ;
} ) , Promise . reject ( new Error ( ) ) ) ;
} catch ( error ) {
if ( error . code !== 'PROFILE_NOT_AVAILABLE' ) {
logger . error ( ` Failed to fetch profile for ' ${ actor . name } ': ${ error . message } ` ) ;
}
}
return null ;
} ) ;
return profiles . filter ( Boolean ) ;
}
2020-05-14 02:26:05 +00:00
async function scrapeActors ( actorNames ) {
const baseActors = toBaseActors ( actorNames ) ;
const sources = argv . sources || config . profiles || Object . keys ( scrapers . actors ) ;
const siteSlugs = sources . flat ( ) ;
const [ networks , sites , existingActorEntries ] = await Promise . all ( [
knex ( 'networks' ) . whereIn ( 'slug' , siteSlugs ) ,
2020-05-18 01:34:41 +00:00
knex ( 'sites' )
. select (
'sites.*' ,
'networks.name as network_name' , 'networks.slug as network_slug' , 'networks.url as network_url' , 'networks.description as network_description' , 'networks.parameters as network_parameters' ,
)
. whereIn ( 'sites.slug' , siteSlugs )
. leftJoin ( 'networks' , 'sites.network_id' , 'networks.id' ) ,
2020-05-14 02:26:05 +00:00
knex ( 'actors' )
. select ( [ 'id' , 'name' , 'slug' ] )
. whereIn ( 'slug' , baseActors . map ( baseActor => baseActor . slug ) )
. whereNull ( 'network_id' ) ,
] ) ;
const existingActorEntriesBySlug = existingActorEntries . reduce ( ( acc , actorEntry ) => ( { ... acc , [ actorEntry . slug ] : actorEntry } ) , { } ) ;
2020-05-18 01:22:03 +00:00
const networksBySlug = networks . reduce ( ( acc , network ) => ( { ... acc , [ network . slug ] : network } ) , { } ) ;
2020-05-18 01:34:41 +00:00
const sitesBySlug = sites . reduce ( ( acc , site ) => ( { ... acc , [ site . slug ] : curateSite ( site ) } ) , { } ) ;
2020-05-14 02:26:05 +00:00
const newBaseActors = baseActors . filter ( baseActor => ! existingActorEntriesBySlug [ baseActor . slug ] ) ;
const [ batchId ] = newBaseActors . length > 0 ? await knex ( 'batches' ) . insert ( { comment : null } ) . returning ( 'id' ) : [ null ] ;
const curatedActorEntries = batchId && curateActorEntries ( newBaseActors , batchId ) ;
const newActorEntries = batchId && await knex ( 'actors' ) . insert ( curatedActorEntries ) . returning ( [ 'id' , 'name' , 'slug' ] ) ;
2020-05-15 02:40:59 +00:00
const actors = existingActorEntries . concat ( Array . isArray ( newActorEntries ) ? newActorEntries : [ ] ) ;
// TODO: don't fetch existing profiles unless --force is used
const profilesPerActor = await Promise . map (
actors ,
async actor => scrapeProfiles ( actor , sources , networksBySlug , sitesBySlug ) ,
{ concurrency : 10 } ,
) ;
2020-05-17 23:22:56 +00:00
const curatedProfiles = await Promise . all ( profilesPerActor . flat ( ) . map ( profile => curateProfile ( profile ) ) ) ;
const profiles = curatedProfiles . filter ( Boolean ) ;
2020-05-16 02:36:45 +00:00
2020-05-17 02:59:09 +00:00
if ( argv . inspect ) {
console . log ( profiles ) ;
}
if ( argv . save ) {
const profilesWithAvatarIds = await associateAvatars ( profiles ) ;
await upsertProfiles ( profilesWithAvatarIds ) ;
await interpolateProfiles ( actors ) ;
}
2020-05-18 01:22:03 +00:00
return profiles ;
2020-05-13 21:17:39 +00:00
}
2020-05-13 00:56:20 +00:00
async function getOrCreateActors ( baseActors , batchId ) {
2020-05-14 02:26:05 +00:00
const existingActors = await knex ( 'actors' )
. select ( 'id' , 'alias_for' , 'name' , 'slug' , 'network_id' )
. whereIn ( 'slug' , baseActors . map ( baseActor => baseActor . slug ) )
. whereNull ( 'network_id' )
. orWhereIn ( [ 'slug' , 'network_id' ] , baseActors . map ( baseActor => [ baseActor . slug , baseActor . network . id ] ) ) ;
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
const existingActorSlugs = existingActors . reduce ( ( acc , actor ) => ( {
... acc ,
[ actor . network _id ] : {
... acc [ actor . network _id ] ,
[ actor . slug ] : true ,
} ,
} ) , { } ) ;
const uniqueBaseActors = baseActors . filter ( baseActor => ! existingActorSlugs [ baseActor . network . id ] ? . [ baseActor . slug ] && ! existingActorSlugs . null ? . [ baseActor . slug ] ) ;
const curatedActorEntries = curateActorEntries ( uniqueBaseActors , batchId ) ;
const newActors = await knex ( 'actors' ) . insert ( curatedActorEntries , [ 'id' , 'alias_for' , 'name' , 'slug' , 'network_id' ] ) ;
if ( Array . isArray ( newActors ) ) {
return newActors . concat ( existingActors ) ;
}
return existingActors ;
2020-03-26 02:32:07 +00:00
}
2020-05-13 00:56:20 +00:00
async function associateActors ( releases , batchId ) {
2020-05-14 02:26:05 +00:00
const baseActorsByReleaseId = releases . reduce ( ( acc , release ) => {
if ( release . actors ) {
acc [ release . id ] = toBaseActors ( release . actors , release ) ;
}
return acc ;
} , { } ) ;
const baseActors = Object . values ( baseActorsByReleaseId ) . flat ( ) ;
if ( baseActors . length === 0 ) {
2020-05-16 02:36:45 +00:00
return null ;
2020-05-14 02:26:05 +00:00
}
2020-05-17 23:22:56 +00:00
const baseActorsBySlug = baseActors . reduce ( ( acc , baseActor ) => ( {
2020-05-14 02:26:05 +00:00
... acc ,
2020-05-17 23:22:56 +00:00
[ baseActor . slug ] : baseActor ,
2020-05-14 02:26:05 +00:00
} ) , { } ) ;
2020-05-17 23:22:56 +00:00
const uniqueBaseActors = Object . values ( baseActorsBySlug ) ;
2020-05-14 02:26:05 +00:00
const actors = await getOrCreateActors ( uniqueBaseActors , batchId ) ;
2020-05-17 23:22:56 +00:00
const actorIdsBySlug = actors . reduce ( ( acc , actor ) => ( {
2020-05-14 02:26:05 +00:00
... acc ,
2020-05-17 23:22:56 +00:00
[ actor . slug ] : actor . alias _for || actor . id ,
2020-05-14 02:26:05 +00:00
} ) , { } ) ;
const releaseActorAssociations = Object . entries ( baseActorsByReleaseId )
. map ( ( [ releaseId , releaseActors ] ) => releaseActors
. map ( releaseActor => ( {
release _id : releaseId ,
2020-05-17 23:22:56 +00:00
actor _id : actorIdsBySlug [ releaseActor . slug ] ,
2020-05-14 02:26:05 +00:00
} ) ) )
. flat ( ) ;
await knex . raw ( ` ${ knex ( 'releases_actors' ) . insert ( releaseActorAssociations ) . toString ( ) } ON CONFLICT DO NOTHING; ` ) ;
2020-05-16 02:36:45 +00:00
return actors ;
2019-11-11 02:20:00 +00:00
}
2019-11-10 03:20:22 +00:00
module . exports = {
2020-05-14 02:26:05 +00:00
associateActors ,
scrapeActors ,
2019-11-10 03:20:22 +00:00
} ;