'use strict'; const config = require('config'); const Promise = require('bluebird'); const moment = require('moment'); const blake2 = require('blake2'); const DOMPurify = require('dompurify'); const { JSDOM } = require('jsdom'); const { window } = new JSDOM(''); const domPurify = DOMPurify(window); // const logger = require('./logger')(__filename); const knex = require('./knex'); const scrapers = require('./scrapers/scrapers').actors; const argv = require('./argv'); const include = require('./utils/argv-include')(argv); const logger = require('./logger')(__filename); const { toBaseReleases } = require('./deep'); const { associateAvatars } = require('./media'); const slugify = require('./utils/slugify'); const capitalize = require('./utils/capitalize'); const resolvePlace = require('./utils/resolve-place'); const hairColors = { 'jet-black': 'black', 'red-head': 'red', 'soft-black': 'black', black: 'black', blonde: 'blonde', blondie: 'blonde', brown: 'brown', brunette: 'brown', fair: 'blonde', raven: 'black', red: 'red', redhead: 'red', blue: 'blue', green: 'green', purple: 'purple', pink: 'pink', }; const eyeColors = { blue: 'blue', brown: 'brown', dark: 'brown', gray: 'gray', green: 'green', grey: 'gray', hazel: 'hazel', }; const ethnicities = { 'african american': 'black', 'african-american': 'black', 'native american': 'native american', african: 'black', aravic: 'arabic', asian: 'asian', black: 'black', caucasian: 'white', european: 'white', hispanic: 'latin', indian: 'indian', japanese: 'japanese', latin: 'latin', latina: 'latina', latino: 'latino', white: 'white', }; function getMostFrequent(items) { const { mostFrequent } = items.reduce((acc, item) => { const slug = slugify(item); acc.counts[slug] = (acc.counts[slug] || 0) + 1; if (!acc.mostFrequent || acc.counts[slug] > acc.counts[slugify(acc.mostFrequent)]) { acc.mostFrequent = item; } return acc; }, { counts: {}, mostFrequent: null, }); return mostFrequent; } function getMostFrequentDate(dates) { const year = getMostFrequent(dates.map(dateX => dateX.getFullYear())); const month = getMostFrequent(dates.map(dateX => dateX.getMonth())); const date = getMostFrequent(dates.map(dateX => dateX.getDate())); if (year && month && date) { return moment({ year, month, date }).toDate(); } return null; } function getLongest(items) { return items.sort((itemA, itemB) => itemB.length - itemA.length)[0] || null; } function getAverage(items) { return Math.round(items.reduce((acc, item) => acc + item, 0) / items.length) || null; } function toBaseActors(actorsOrNames, release) { return actorsOrNames.map((actorOrName) => { const name = capitalize(actorOrName.name || actorOrName); const slug = slugify(name); const baseActor = { name, slug, entity: release?.site?.network || release?.entity?.parent || release?.entity || null, }; if (actorOrName.name) { return { ...actorOrName, ...baseActor, }; } return baseActor; }); } function curateActor(actor, withDetails = false) { if (!actor) { return null; } const curatedActor = { id: actor.id, name: actor.name, slug: actor.slug, gender: actor.gender, entityId: actor.entity_id, aliasFor: actor.alias_for, dateOfBirth: actor.date_of_birth, birthCountry: actor.birth_country_alpha2, ...(withDetails && { alias: actor.alias && { id: actor.alias.id, name: actor.alias.name, slug: actor.slug, gender: actor.alias.gender, }, entity: actor.entity && { id: actor.entity.id, name: actor.entity.name, slug: actor.entity.slug, }, dateOfDeath: actor.date_of_death, cup: actor.cup, bust: actor.bust, waist: actor.waist, hip: actor.hip, naturalBoobs: actor.natural_boobs, height: actor.height, weight: actor.weight, eyes: actor.eyes, hairColor: actor.hair_color, hasTattoos: actor.has_tattoos, hasPiercings: actor.has_piercings, tattoos: actor.tattoos, piercings: actor.piercings, description: actor.description, placeOfBirth: actor.birth_country && { country: { alpha2: actor.birth_country.alpha2, name: actor.birth_country.name, alias: actor.birth_country.alias, }, state: actor.birth_state, city: actor.birth_city, }, placeOfResidence: actor.residence_country && { country: { alpha2: actor.residence_country.alpha2, name: actor.residence_country.name, alias: actor.residence_country.alias, }, state: actor.residence_state, city: actor.residence_city, }, avatar: actor.avatar && { id: actor.avatar.id, path: actor.avatar.path, width: actor.avatar.width, height: actor.avatar.height, size: actor.avatar.size, source: actor.avatar.source, }, }), }; return curatedActor; } function curateActorEntry(baseActor, batchId) { return { name: baseActor.name, slug: baseActor.slug, entity_id: null, batch_id: batchId, }; } function curateActorEntries(baseActors, batchId) { return baseActors.map(baseActor => curateActorEntry(baseActor, batchId)); } function curateProfileEntry(profile) { const curatedProfileEntry = { ...(profile.update !== false && { id: profile.update }), actor_id: profile.id, entity_id: profile.entity?.id || null, date_of_birth: profile.dateOfBirth, date_of_death: profile.dateOfDeath, gender: profile.gender, ethnicity: profile.ethnicity, description: profile.description, description_hash: profile.descriptionHash, birth_city: profile.placeOfBirth?.city || null, birth_state: profile.placeOfBirth?.state || null, birth_country_alpha2: profile.placeOfBirth?.country || null, residence_city: profile.placeOfResidence?.city || null, residence_state: profile.placeOfResidence?.state || null, residence_country_alpha2: profile.placeOfResidence?.country || null, cup: profile.cup, bust: profile.bust, waist: profile.waist, hip: profile.hip, natural_boobs: profile.naturalBoobs, height: profile.height, weight: profile.weight, hair_color: profile.hairColor, eyes: profile.eyes, has_tattoos: profile.hasTattoos, has_piercings: profile.hasPiercings, piercings: profile.piercings, tattoos: profile.tattoos, avatar_media_id: profile.avatarMediaId || null, }; return curatedProfileEntry; } async function curateProfile(profile) { if (!profile) { return null; } try { const curatedProfile = { id: profile.id, name: profile.name, avatar: profile.avatar, scraper: profile.scraper, entity: profile.entity, update: profile.update, }; curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null; const hasher = curatedProfile.description && blake2 .createHash('blake2b', { digestLength: 24 }) .update(Buffer.from(slugify(curatedProfile.description))); curatedProfile.descriptionHash = curatedProfile.description && hasher.digest('hex'); curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null; curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null; curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null; curatedProfile.tattoos = profile.tattoos?.trim() || null; curatedProfile.piercings = profile.piercings?.trim() || null; curatedProfile.gender = (/female/i.test(profile.gender) && 'female') || (/shemale/i.test(profile.gender) && 'transsexual') || (/male/i.test(profile.gender) && 'male') || (/trans/i.test(profile.gender) && 'transsexual') || null; const dateOfBirth = profile.dateOfBirth || profile.birthdate; curatedProfile.dateOfBirth = (!Number.isNaN(Number(dateOfBirth)) // possibly valid date && new Date() - dateOfBirth > 567648000000 // over 18 && dateOfBirth) || null; curatedProfile.dateOfDeath = Number.isNaN(Number(profile.dateOfDeath)) ? null : profile.dateOfDeath; curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null; curatedProfile.bust = Number(profile.bust) || profile.bust?.match?.(/\d+/)?.[0] || null; curatedProfile.waist = Number(profile.waist) || profile.waist?.match?.(/\d+/)?.[0] || null; curatedProfile.hip = Number(profile.hip) || profile.hip?.match?.(/\d+/)?.[0] || null; curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null; curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null; curatedProfile.naturalBoobs = typeof profile.naturalBoobs === 'boolean' ? profile.naturalBoobs : null; curatedProfile.hasTattoos = typeof profile.hasTattoos === 'boolean' ? profile.hasTattoos : null; curatedProfile.hasPiercings = typeof profile.hasPiercings === 'boolean' ? profile.hasPiercings : null; if (argv.resolvePlace) { const [placeOfBirth, placeOfResidence] = await Promise.all([ resolvePlace(profile.birthPlace), resolvePlace(profile.residencePlace), ]); curatedProfile.placeOfBirth = placeOfBirth; curatedProfile.placeOfResidence = placeOfResidence; } if (!curatedProfile.placeOfBirth && curatedProfile.nationality) { const country = await knex('countries') .where('nationality', 'ilike', `%${curatedProfile.nationality}%`) .orderBy('priority', 'desc') .first(); curatedProfile.placeOfBirth = { country: country.alpha2, }; } curatedProfile.social = Array.isArray(profile.social) ? profile.social.map((social) => { try { const { href } = new URL(social); return href; } catch (error) { logger.warn(`Profile scraper for '${profile.entity.name}' returned invalid social link: ${social}`); return null; } }).filter(Boolean) : []; curatedProfile.releases = toBaseReleases(profile.releases); if (profile.ethnicity && !curatedProfile.ethnicity) logger.warn(`Unrecognized ethnicity returned by '${profile.entity.name}' scraper: ${profile.ethnicity}`); if ((profile.hairColor || profile.hair) && !curatedProfile.hairColor) logger.warn(`Unrecognized hair color returned by '${profile.entity.name}' scraper: ${profile.hairColor || profile.hair}`); if (profile.eyes && !curatedProfile.eyes) logger.warn(`Unrecognized eye color returned by '${profile.entity.name}' scraper: ${profile.eyes}`); return curatedProfile; } catch (error) { logger.error(`Failed to curate '${profile.name}': ${error.message}`); return null; } } async function interpolateProfiles(actors) { const profiles = await knex('actors_profiles') .select(['actors_profiles.*', 'media.width as avatar_width', 'media.height as avatar_height', 'media.size as avatar_size']) .whereIn('actor_id', actors.map(actor => actor.id)) .leftJoin('media', 'actors_profiles.avatar_media_id', 'media.id'); const profilesByActorId = profiles.reduce((acc, profile) => ({ ...acc, [profile.actor_id]: [ ...(acc[profile.actor_id] || []), profile, ], }), {}); const interpolatedProfiles = Object.entries(profilesByActorId).map(([actorId, actorProfiles]) => { // group values from each profile const valuesByProperty = actorProfiles.reduce((acc, profile) => Object .entries(profile) .reduce((profileAcc, [property, value]) => ({ ...profileAcc, [property]: [ ...(acc[property] || []), ...(value === null ? [] : Array.from({ length: profile.priority }, () => value)), // multiply by priority, increasing the odds of being the most frequent value ], }), { // bundle location values so they can be assessed together, to ensure the most frequent city is in the most frequent state is in most frequent country origin: [...acc.origin || [], { ...(profile.birth_country_alpha2 && { country: profile.birth_country_alpha2 }), ...(profile.birth_state && { state: profile.birth_state }), ...(profile.birth_city && { city: profile.birth_city }), }].filter(location => Object.keys(location).length > 0), residence: [...acc.residence || [], { ...(profile.residence_country_alpha2 && { country: profile.residence_country_alpha2 }), ...(profile.residence_state && { state: profile.residence_state }), ...(profile.residence_city && { city: profile.residence_city }), }].filter(location => Object.keys(location).length > 0), }), {}); const avatars = actorProfiles.map(profile => profile.avatar_media_id && ({ id: profile.avatar_media_id, width: profile.avatar_width, height: profile.avatar_height, size: profile.avatar_size, })).filter(Boolean); const mostFrequentValues = [ 'gender', 'ethnicity', 'cup', 'bust', 'waist', 'hip', 'natural_boobs', 'height', 'hair_color', 'eyes', 'has_tattoos', 'has_piercings', ].reduce((acc, property) => ({ ...acc, [property]: getMostFrequent(valuesByProperty[property]), }), {}); const profile = { id: actorId, ...mostFrequentValues, }; profile.date_of_birth = getMostFrequentDate(valuesByProperty.date_of_birth); profile.date_of_death = getMostFrequentDate(valuesByProperty.date_of_death); // ensure most frequent country, city and state match up profile.birth_country_alpha2 = getMostFrequent(valuesByProperty.origin.map(location => location.country)); const remainingOriginCountries = valuesByProperty.origin.filter(location => location.country === profile.birth_country_alpha2); profile.birth_state = getMostFrequent(remainingOriginCountries.map(location => location.state)); const remainingOriginStates = remainingOriginCountries.filter(location => !profile.birth_state || location.state === profile.birth_state); profile.birth_city = getMostFrequent(remainingOriginStates.map(location => location.city)); profile.residence_country_alpha2 = getMostFrequent(valuesByProperty.residence.map(location => location.country)); const remainingResidenceCountries = valuesByProperty.residence.filter(location => location.country === profile.residence_country_alpha2); profile.residence_state = getMostFrequent(remainingResidenceCountries.map(location => location.state)); const remainingResidenceStates = remainingResidenceCountries.filter(location => !profile.residence_state || location.state === profile.residence_state); profile.residence_city = getMostFrequent(remainingResidenceStates.map(location => location.city)); profile.weight = getAverage(valuesByProperty.weight); profile.tattoos = getLongest(valuesByProperty.tattoos); profile.piercings = getLongest(valuesByProperty.piercings); profile.avatar_media_id = avatars.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; return profile; }); const transaction = await knex.transaction(); const queries = interpolatedProfiles.map(profile => knex('actors') .where('id', profile.id) .update(profile) .transacting(transaction)); await Promise.all(queries) .then(transaction.commit) .catch(transaction.rollback); } async function upsertProfiles(profiles) { const newProfileEntries = profiles.filter(profile => !profile.update).map(profile => curateProfileEntry(profile)); const updatingProfileEntries = profiles.filter(profile => profile.update).map(profile => curateProfileEntry(profile)); if (newProfileEntries.length > 0) { await knex.batchInsert('actors_profiles', newProfileEntries); logger.info(`Saved ${newProfileEntries.length} actor profiles`); } if (argv.force && updatingProfileEntries.length > 0) { const transaction = await knex.transaction(); const queries = updatingProfileEntries.map(profileEntry => knex('actors_profiles') .where('id', profileEntry.id) .update(profileEntry) .returning(['id', 'actor_id']) .transacting(transaction)); await Promise.all(queries) .then(transaction.commit) .catch(transaction.rollback); logger.info(`Updated ${updatingProfileEntries.length} new actor profiles`); } } async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) { const profiles = Promise.map(sources, async (source) => { try { // config may group sources to try until success return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => { try { const scraper = scrapers[scraperSlug]; const context = { ...entitiesBySlug[scraperSlug], // legacy site: entitiesBySlug[scraperSlug] || null, network: entitiesBySlug[scraperSlug] || null, entity: entitiesBySlug[scraperSlug] || null, scraper: scraperSlug, }; const label = context.entity?.name; if (!scraper?.fetchProfile) { logger.warn(`No profile profile scraper available for ${scraperSlug}`); throw new Error(`No profile profile scraper available for ${scraperSlug}`); } if (!context.entity) { logger.warn(`No entity found for ${scraperSlug}`); throw new Error(`No entity found for ${scraperSlug}`); } const existingProfile = existingProfilesByActorEntityId[actor.id]?.[context.entity?.id || null]; if (existingProfile && !argv.force) { logger.verbose(`Found existing profile for '${actor.name}' on '${label}', use --force to scrape again`); return null; } logger.verbose(`Searching profile for '${actor.name}' on '${label}'`); const profile = await scraper.fetchProfile(actor.name, context, include); if (!profile || typeof profile === 'number') { // scraper returns HTTP code on request failure logger.verbose(`Profile for '${actor.name}' not available on ${label}, scraper returned ${profile}`); throw Object.assign(new Error(`Profile for '${actor.name}' not available on ${label}`), { code: 'PROFILE_NOT_AVAILABLE' }); } logger.verbose(`Found profile for '${actor.name}' on '${label}'`); return await curateProfile({ ...actor, ...profile, ...context, update: existingProfile?.id || false, }); } catch (error) { if (error.code !== 'PROFILE_NOT_AVAILABLE') { logger.error(`Failed to fetch profile for '${actor.name}' from '${scraperSlug}': ${error.message}`); } // throw error to try next source throw error; } }), Promise.reject(new Error())); } catch (error) { if (error.code !== 'PROFILE_NOT_AVAILABLE') { logger.error(`Failed to fetch profile for '${actor.name}': ${error.message}`); } } return null; }); return profiles.filter(Boolean); } async function scrapeActors(actorNames) { const baseActors = toBaseActors(actorNames); const sources = argv.sources || config.profiles || Object.keys(scrapers.actors); const entitySlugs = sources.flat(); const [entities, existingActorEntries] = await Promise.all([ knex('entities') .select(knex.raw('entities.*, row_to_json(parents) as parent')) .whereIn('entities.slug', entitySlugs) .leftJoin('entities as parents', 'parents.id', 'entities.parent_id') .orderBy('entities.type'), knex('actors') .select(['id', 'name', 'slug']) .modify((queryBuilder) => { if (actorNames.length > 0) { queryBuilder.whereIn('slug', baseActors.map(baseActor => baseActor.slug)); } }) .whereNull('alias_for'), ]); const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: entity }), {}); const existingActorEntriesBySlug = existingActorEntries.reduce((acc, actorEntry) => ({ ...acc, [actorEntry.slug]: actorEntry }), {}); const newBaseActors = baseActors.filter(baseActor => !existingActorEntriesBySlug[baseActor.slug]); const [batchId] = newBaseActors.length > 0 ? await knex('batches').insert({ comment: null }).returning('id') : [null]; const curatedActorEntries = batchId && curateActorEntries(newBaseActors, batchId); const newActorEntries = batchId && await knex('actors').insert(curatedActorEntries).returning(['id', 'name', 'slug']); const actors = existingActorEntries.concat(Array.isArray(newActorEntries) ? newActorEntries : []); const existingProfiles = await knex('actors_profiles').whereIn('actor_id', actors.map(actor => actor.id)); const existingProfilesByActorEntityId = existingProfiles.reduce((acc, profile) => ({ ...acc, [profile.actor_id]: { ...acc[profile.actor_id], [profile.entity_id]: profile, }, }), {}); const profilesPerActor = await Promise.map( actors, async actor => scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId), { concurrency: 10 }, ); const profiles = profilesPerActor.flat().filter(Boolean); logger.info(`Scraped ${profiles.length} profiles`); if (argv.inspect) { console.log(profiles); } if (argv.save) { const profilesWithAvatarIds = await associateAvatars(profiles); await upsertProfiles(profilesWithAvatarIds); await interpolateProfiles(actors); } return profiles; } async function getOrCreateActors(baseActors, batchId) { const existingActors = await knex('actors') .select('id', 'alias_for', 'name', 'slug', 'entity_id') .whereIn('slug', baseActors.map(baseActor => baseActor.slug)) .whereNull('entity_id') .orWhereIn(['slug', 'entity_id'], baseActors.map(baseActor => [baseActor.slug, baseActor.entity.id])); // const existingActorSlugs = new Set(existingActors.map(actor => actor.slug)); const existingActorSlugs = existingActors.reduce((acc, actor) => ({ ...acc, [actor.entity_id]: { ...acc[actor.entity_id], [actor.slug]: true, }, }), {}); const uniqueBaseActors = baseActors.filter(baseActor => !existingActorSlugs[baseActor.entity.id]?.[baseActor.slug] && !existingActorSlugs.null?.[baseActor.slug]); const curatedActorEntries = curateActorEntries(uniqueBaseActors, batchId); const newActors = await knex('actors').insert(curatedActorEntries, ['id', 'alias_for', 'name', 'slug', 'entity_id']); if (Array.isArray(newActors)) { return newActors.concat(existingActors); } return existingActors; } async function associateActors(releases, batchId) { const baseActorsByReleaseId = releases.reduce((acc, release) => { if (release.actors) { acc[release.id] = toBaseActors(release.actors, release); } return acc; }, {}); const baseActors = Object.values(baseActorsByReleaseId).flat(); if (baseActors.length === 0) { return null; } const baseActorsBySlug = baseActors.reduce((acc, baseActor) => ({ ...acc, [baseActor.slug]: baseActor, }), {}); const uniqueBaseActors = Object.values(baseActorsBySlug); const actors = await getOrCreateActors(uniqueBaseActors, batchId); const actorIdsBySlug = actors.reduce((acc, actor) => ({ ...acc, [actor.slug]: actor.alias_for || actor.id, }), {}); const releaseActorAssociations = Object.entries(baseActorsByReleaseId) .map(([releaseId, releaseActors]) => releaseActors .map(releaseActor => ({ release_id: releaseId, actor_id: actorIdsBySlug[releaseActor.slug], }))) .flat(); await knex.raw(`${knex('releases_actors').insert(releaseActorAssociations).toString()} ON CONFLICT DO NOTHING;`); return actors; } async function fetchActor(actorId) { const actor = await knex('actors') .select(knex.raw(` actors.*, row_to_json(entities) as entity, row_to_json(actor_alias) as alias, row_to_json(birth_country) as birth_country, row_to_json(residence_country) as residence_country, row_to_json(media) as avatar `)) .modify((queryBuilder) => { if (Number.isNaN(Number(actorId))) { queryBuilder.where('actors.slug', actorId); return; } queryBuilder.where('actors.id', actorId); }) .leftJoin('actors as actor_alias', 'actor_alias.id', 'actors.alias_for') .leftJoin('entities', 'entities.id', 'actors.entity_id') .leftJoin('countries as birth_country', 'birth_country.alpha2', 'actors.birth_country_alpha2') .leftJoin('countries as residence_country', 'residence_country.alpha2', 'actors.residence_country_alpha2') .leftJoin('media', 'media.id', 'actors.avatar_media_id') .first(); return curateActor(actor, true); } async function searchActors(query) { const actors = await knex .select('*') .from(knex.raw('search_actors(?) as actors', [query])) .limit(10); return actors.map(actor => curateActor(actor)); } module.exports = { associateActors, fetchActor, scrapeActors, searchActors, };