504 lines
18 KiB
JavaScript
504 lines
18 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const Promise = require('bluebird');
|
|
const UrlPattern = require('url-pattern');
|
|
const moment = require('moment');
|
|
|
|
const logger = require('./logger')(__filename);
|
|
const knex = require('./knex');
|
|
const argv = require('./argv');
|
|
const scrapers = require('./scrapers/scrapers');
|
|
const whereOr = require('./utils/where-or');
|
|
const resolvePlace = require('./utils/resolve-place');
|
|
const slugify = require('./utils/slugify');
|
|
const { createMediaDirectory, storePhotos } = require('./media');
|
|
|
|
async function curateActor(actor) {
|
|
const [aliases, avatar, photos, social] = await Promise.all([
|
|
knex('actors').where({ alias_for: actor.id }),
|
|
knex('actors_avatars')
|
|
.where('actor_id', actor.id)
|
|
.join('media', 'media.id', 'actors_avatars.media_id')
|
|
.first(),
|
|
knex('actors_photos')
|
|
.where('actor_id', actor.id)
|
|
.join('media', 'media.id', 'actors_photos.media_id')
|
|
.orderBy('index'),
|
|
knex('actors_social')
|
|
.where('actor_id', actor.id)
|
|
.orderBy('platform', 'desc'),
|
|
]);
|
|
|
|
const curatedActor = {
|
|
id: actor.id,
|
|
gender: actor.gender,
|
|
name: actor.name,
|
|
description: actor.description,
|
|
birthdate: actor.birthdate && new Date(actor.birthdate),
|
|
country: actor.country_alpha2,
|
|
origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
|
|
residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
|
|
ethnicity: actor.ethnicity,
|
|
height: actor.height,
|
|
weight: actor.weight,
|
|
bust: actor.bust,
|
|
waist: actor.waist,
|
|
hip: actor.hip,
|
|
naturalBoobs: actor.natural_boobs,
|
|
aliases: aliases.map(({ name }) => name),
|
|
slug: actor.slug,
|
|
avatar,
|
|
photos,
|
|
hasTattoos: actor.has_tattoos,
|
|
hasPiercings: actor.has_piercings,
|
|
tattoos: actor.tattoos,
|
|
piercings: actor.piercings,
|
|
social,
|
|
scrapedAt: actor.scraped_at,
|
|
};
|
|
|
|
if (curatedActor.birthdate) {
|
|
curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
|
|
}
|
|
|
|
if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
|
|
if (actor.birth_state) curatedActor.origin.state = actor.birth_state;
|
|
|
|
if (actor.birth_country_alpha2) {
|
|
curatedActor.origin.country = {
|
|
alpha2: actor.birth_country_alpha2,
|
|
name: actor.birth_country_name,
|
|
alias: actor.birth_country_alias,
|
|
};
|
|
}
|
|
|
|
if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
|
|
if (actor.residence_state) curatedActor.residence.state = actor.residence_state;
|
|
|
|
if (actor.residence_country_alpha2) {
|
|
curatedActor.residence.country = {
|
|
alpha2: actor.residence_country_alpha2,
|
|
name: actor.residence_country_name,
|
|
alias: actor.residence_country_alias,
|
|
};
|
|
}
|
|
|
|
return curatedActor;
|
|
}
|
|
|
|
function curateActors(releases) {
|
|
return Promise.all(releases.map(async release => curateActor(release)));
|
|
}
|
|
|
|
function curateActorEntry(actor, scraped, scrapeSuccess) {
|
|
const curatedActor = {
|
|
name: actor.name
|
|
.split(' ')
|
|
.map(segment => `${segment.charAt(0).toUpperCase()}${segment.slice(1)}`)
|
|
.join(' '),
|
|
slug: slugify(actor.name),
|
|
birthdate: actor.birthdate,
|
|
description: actor.description,
|
|
gender: actor.gender,
|
|
ethnicity: actor.ethnicity,
|
|
bust: actor.bust,
|
|
waist: actor.waist,
|
|
hip: actor.hip,
|
|
natural_boobs: actor.naturalBoobs,
|
|
height: actor.height,
|
|
weight: actor.weight,
|
|
hair: actor.hair,
|
|
eyes: actor.eyes,
|
|
has_tattoos: actor.hasTattoos,
|
|
has_piercings: actor.hasPiercings,
|
|
tattoos: actor.tattoos,
|
|
piercings: actor.piercings,
|
|
};
|
|
|
|
if (actor.id) {
|
|
curatedActor.id = actor.id;
|
|
}
|
|
|
|
if (actor.birthPlace) {
|
|
curatedActor.birth_city = actor.birthPlace.city;
|
|
curatedActor.birth_state = actor.birthPlace.state;
|
|
curatedActor.birth_country_alpha2 = actor.birthPlace.country;
|
|
}
|
|
|
|
if (actor.residencePlace) {
|
|
curatedActor.residence_city = actor.residencePlace.city;
|
|
curatedActor.residence_state = actor.residencePlace.state;
|
|
curatedActor.residence_country_alpha2 = actor.residencePlace.country;
|
|
}
|
|
|
|
if (scraped) {
|
|
curatedActor.scraped_at = new Date();
|
|
curatedActor.scrape_success = scrapeSuccess;
|
|
}
|
|
|
|
return curatedActor;
|
|
}
|
|
|
|
function curateSocialEntry(url, actorId) {
|
|
const platforms = [
|
|
// links supplied by PH often look like domain.com/domain.com/username
|
|
{
|
|
label: 'twitter',
|
|
pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
|
|
format: username => `https://www.twitter.com/${username}`,
|
|
},
|
|
{
|
|
label: 'youtube',
|
|
pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)',
|
|
format: username => `https://www.youtube.com/channel/${username}`,
|
|
},
|
|
{
|
|
label: 'instagram',
|
|
pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
|
|
format: username => `https://www.instagram.com/${username}`,
|
|
},
|
|
{
|
|
label: 'snapchat',
|
|
pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
|
|
format: username => `https://www.snapchat.com/add/${username}`,
|
|
},
|
|
{
|
|
label: 'tumblr',
|
|
pattern: 'http(s)\\://:username.tumblr.com(*)',
|
|
format: username => `https://${username}.tumblr.com`,
|
|
},
|
|
{
|
|
label: 'onlyfans',
|
|
pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)',
|
|
format: username => `https://www.onlyfans.com/${username}`,
|
|
},
|
|
{
|
|
label: 'fancentro',
|
|
pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)',
|
|
format: username => `https://www.fancentro.com/${username}`,
|
|
},
|
|
{
|
|
label: 'modelhub',
|
|
pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)',
|
|
format: username => `https://www.modelhub.com/${username}`,
|
|
},
|
|
];
|
|
|
|
const match = platforms.reduce((acc, platform) => {
|
|
if (acc) return acc;
|
|
|
|
const patternMatch = new UrlPattern(platform.pattern).match(url);
|
|
|
|
if (patternMatch) {
|
|
return {
|
|
platform: platform.label,
|
|
original: url,
|
|
username: patternMatch.username,
|
|
url: platform.format ? platform.format(patternMatch.username) : url,
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}, null) || { url };
|
|
|
|
return {
|
|
url: match.url,
|
|
platform: match.platform,
|
|
actor_id: actorId,
|
|
};
|
|
}
|
|
|
|
async function curateSocialEntries(urls, actorId) {
|
|
if (!urls) {
|
|
return [];
|
|
}
|
|
|
|
const existingSocialLinks = await knex('actors_social').where('actor_id', actorId);
|
|
|
|
return urls.reduce((acc, url) => {
|
|
const socialEntry = curateSocialEntry(url, actorId);
|
|
|
|
if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
|
|
// prevent duplicates
|
|
return acc;
|
|
}
|
|
|
|
return [...acc, socialEntry];
|
|
}, []);
|
|
}
|
|
|
|
async function fetchActors(queryObject, limit = 100) {
|
|
const releases = await knex('actors')
|
|
.select(
|
|
'actors.*',
|
|
'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
|
|
'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
|
|
)
|
|
.leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
|
|
.leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
|
|
.orderBy(['actors.name', 'actors.gender'])
|
|
.where(builder => whereOr(queryObject, 'actors', builder))
|
|
.limit(limit);
|
|
|
|
return curateActors(releases);
|
|
}
|
|
|
|
async function storeSocialLinks(urls, actorId) {
|
|
const curatedSocialEntries = await curateSocialEntries(urls, actorId);
|
|
|
|
await knex('actors_social').insert(curatedSocialEntries);
|
|
}
|
|
|
|
async function storeActor(actor, scraped = false, scrapeSuccess = false) {
|
|
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
|
|
|
|
const [actorEntry] = await knex('actors')
|
|
.insert(curatedActor)
|
|
.returning('*');
|
|
|
|
await storeSocialLinks(actor.social, actorEntry.id);
|
|
|
|
if (actor.avatars) {
|
|
await createMediaDirectory('actors', `${actorEntry.slug}/`);
|
|
await storePhotos(actor.avatars, {
|
|
domain: 'actor',
|
|
role: 'photo',
|
|
primaryRole: 'avatar',
|
|
targetId: actorEntry.id,
|
|
subpath: `${actorEntry.slug}/`,
|
|
naming: 'timestamp',
|
|
}, actorEntry.name);
|
|
}
|
|
|
|
logger.info(`Added new entry for actor '${actor.name}'`);
|
|
|
|
return actorEntry;
|
|
}
|
|
|
|
async function updateActor(actor, scraped = false, scrapeSuccess = false) {
|
|
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
|
|
|
|
const [actorEntry] = await knex('actors')
|
|
.where({ id: actor.id })
|
|
.update(curatedActor)
|
|
.returning('*');
|
|
|
|
await storeSocialLinks(actor.social, actor.id);
|
|
|
|
logger.info(`Updated entry for actor '${actor.name}'`);
|
|
|
|
return actorEntry;
|
|
}
|
|
|
|
async function mergeProfiles(profiles, actor) {
|
|
if (profiles.filter(Boolean).length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const mergedProfile = profiles.reduce((prevProfile, profile) => {
|
|
if (profile === null) {
|
|
return prevProfile;
|
|
}
|
|
|
|
return {
|
|
id: actor ? actor.id : null,
|
|
name: actor ? actor.name : (prevProfile.name || profile.name),
|
|
description: prevProfile.description || profile.description,
|
|
gender: prevProfile.gender || profile.gender,
|
|
birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
|
|
birthPlace: prevProfile.birthPlace || profile.birthPlace,
|
|
residencePlace: prevProfile.residencePlace || profile.residencePlace,
|
|
nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available
|
|
ethnicity: prevProfile.ethnicity || profile.ethnicity,
|
|
bust: prevProfile.bust || profile.bust,
|
|
waist: prevProfile.waist || profile.waist,
|
|
hip: prevProfile.hip || profile.hip,
|
|
naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
|
|
height: prevProfile.height || profile.height,
|
|
weight: prevProfile.weight || profile.weight,
|
|
hair: prevProfile.hair || profile.hair,
|
|
eyes: prevProfile.eyes || profile.eyes,
|
|
hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
|
|
hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
|
|
piercings: prevProfile.piercings || profile.piercings,
|
|
tattoos: prevProfile.tattoos || profile.tattoos,
|
|
social: prevProfile.social.concat(profile.social || []),
|
|
avatars: prevProfile.avatars.concat(profile.avatar ? [profile.avatar] : []), // don't flatten fallbacks
|
|
releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
|
|
};
|
|
}, {
|
|
social: [],
|
|
avatars: [],
|
|
releases: [],
|
|
});
|
|
|
|
const [birthPlace, residencePlace] = await Promise.all([
|
|
resolvePlace(mergedProfile.birthPlace),
|
|
resolvePlace(mergedProfile.residencePlace),
|
|
]);
|
|
|
|
mergedProfile.birthPlace = birthPlace;
|
|
mergedProfile.residencePlace = residencePlace;
|
|
|
|
if (!mergedProfile.birthPlace && mergedProfile.nationality) {
|
|
const country = await knex('countries')
|
|
.where('nationality', 'ilike', `%${mergedProfile.nationality}%`)
|
|
.orderBy('priority', 'desc')
|
|
.first();
|
|
|
|
mergedProfile.birthPlace = {
|
|
country: country.alpha2,
|
|
};
|
|
}
|
|
|
|
return mergedProfile;
|
|
}
|
|
|
|
async function scrapeActors(actorNames) {
|
|
return Promise.map(actorNames || argv.actors, async (actorName) => {
|
|
try {
|
|
const actorSlug = slugify(actorName);
|
|
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
|
|
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
|
|
|
|
const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested
|
|
|
|
const profiles = await Promise.map(finalSources, async (source) => {
|
|
// const [scraperSlug, scraper] = source;
|
|
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
|
|
|
|
try {
|
|
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
|
|
if (!scraper) {
|
|
logger.warn(`No profile profile scraper available for ${scraperSlug}`);
|
|
throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
|
|
}
|
|
|
|
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
|
|
|
|
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
|
|
|
|
if (profile) {
|
|
logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);
|
|
|
|
return {
|
|
...profile,
|
|
name: actorName,
|
|
scraper: scraperSlug,
|
|
};
|
|
}
|
|
|
|
logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}`);
|
|
throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false });
|
|
}), Promise.reject(new Error()));
|
|
} catch (error) {
|
|
if (error.warn !== false) {
|
|
logger.warn(`Error in scraper ${source}: ${error.message}`);
|
|
// logger.error(error.stack);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
});
|
|
|
|
const profile = await mergeProfiles(profiles, actorEntry);
|
|
|
|
if (argv.inspect) {
|
|
console.log(profile);
|
|
logger.info(`Found ${profile.releases.length} releases for ${actorName}`);
|
|
}
|
|
|
|
if (profile === null) {
|
|
logger.warn(`Could not find profile for actor '${actorName}'`);
|
|
|
|
if (argv.save && !actorEntry) {
|
|
await storeActor({ name: actorName }, false, false);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
if (argv.save) {
|
|
if (actorEntry && profile) {
|
|
await createMediaDirectory('actors', `${actorEntry.slug}/`);
|
|
|
|
await Promise.all([
|
|
updateActor(profile, true, true),
|
|
// storeAvatars(profile, actorEntry),
|
|
storePhotos(profile.avatars, {
|
|
domain: 'actor',
|
|
role: 'photo',
|
|
primaryRole: 'avatar',
|
|
targetId: actorEntry.id,
|
|
subpath: `${actorEntry.slug}/`,
|
|
naming: 'timestamp',
|
|
}, actorEntry.name),
|
|
]);
|
|
|
|
return profile;
|
|
}
|
|
|
|
await storeActor(profile, true, true);
|
|
}
|
|
|
|
return profile;
|
|
} catch (error) {
|
|
console.log(error);
|
|
logger.warn(`${actorName}: ${error}`);
|
|
|
|
return null;
|
|
}
|
|
}, {
|
|
concurrency: 3,
|
|
});
|
|
}
|
|
|
|
async function scrapeBasicActors() {
|
|
const basicActors = await knex('actors').where('scraped_at', null);
|
|
|
|
return scrapeActors(basicActors.map(actor => actor.name));
|
|
}
|
|
|
|
async function associateActors(mappedActors, releases) {
|
|
const [existingActorEntries, existingAssociationEntries] = await Promise.all([
|
|
knex('actors')
|
|
.whereIn('name', Object.values(mappedActors).map(actor => actor.name))
|
|
.orWhereIn('slug', Object.keys(mappedActors)),
|
|
knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
|
|
]);
|
|
|
|
const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
|
|
try {
|
|
const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
|
|
|| await storeActor(actor);
|
|
|
|
// if a scene
|
|
return Array.from(actor.releaseIds)
|
|
.map(releaseId => ({
|
|
release_id: releaseId,
|
|
actor_id: actorEntry.id,
|
|
}))
|
|
.filter(association => !existingAssociationEntries
|
|
// remove associations already in database
|
|
.some(associationEntry => associationEntry.actor_id === association.actor_id
|
|
&& associationEntry.release_id === association.release_id));
|
|
} catch (error) {
|
|
logger.error(actor.name, error);
|
|
return null;
|
|
}
|
|
});
|
|
|
|
await knex('releases_actors').insert(associations.filter(association => association).flat());
|
|
|
|
// basic actor scraping is failure prone, don't run together with actor association
|
|
// await scrapebasicactors(),
|
|
}
|
|
|
|
module.exports = {
|
|
associateActors,
|
|
fetchActors,
|
|
scrapeActors,
|
|
scrapeBasicActors,
|
|
};
|