traxxx/src/actors.js

'use strict';

const config = require('config');
const Promise = require('bluebird');
const UrlPattern = require('url-pattern');
const moment = require('moment');

const logger = require('./logger')(__filename);
const knex = require('./knex');
const argv = require('./argv');
const scrapers = require('./scrapers/scrapers');
const whereOr = require('./utils/where-or');
const resolvePlace = require('./utils/resolve-place');
const slugify = require('./utils/slugify');
const { createMediaDirectory, storePhotos } = require('./media');

async function curateActor(actor) {
    const [aliases, avatar, photos, social] = await Promise.all([
        knex('actors').where({ alias_for: actor.id }),
        knex('actors_avatars')
            .where('actor_id', actor.id)
            .join('media', 'media.id', 'actors_avatars.media_id')
            .first(),
        knex('actors_photos')
            .where('actor_id', actor.id)
            .join('media', 'media.id', 'actors_photos.media_id')
            .orderBy('index'),
        knex('actors_social')
            .where('actor_id', actor.id)
            .orderBy('platform', 'desc'),
    ]);

    const curatedActor = {
        id: actor.id,
        gender: actor.gender,
        name: actor.name,
        description: actor.description,
        birthdate: actor.birthdate && new Date(actor.birthdate),
        country: actor.country_alpha2,
        origin: (actor.birth_city || actor.birth_state || actor.birth_country_alpha2) ? {} : null,
        residence: (actor.residence_city || actor.residence_state || actor.residence_country_alpha2) ? {} : null,
        ethnicity: actor.ethnicity,
        height: actor.height,
        weight: actor.weight,
        bust: actor.bust,
        waist: actor.waist,
        hip: actor.hip,
        naturalBoobs: actor.natural_boobs,
        aliases: aliases.map(({ name }) => name),
        slug: actor.slug,
        avatar,
        photos,
        hasTattoos: actor.has_tattoos,
        hasPiercings: actor.has_piercings,
        tattoos: actor.tattoos,
        piercings: actor.piercings,
        social,
        scrapedAt: actor.scraped_at,
    };

    if (curatedActor.birthdate) {
        curatedActor.age = moment().diff(curatedActor.birthdate, 'years');
    }

    if (actor.birth_city) curatedActor.origin.city = actor.birth_city;
    if (actor.birth_state) curatedActor.origin.state = actor.birth_state;

    if (actor.birth_country_alpha2) {
        curatedActor.origin.country = {
            alpha2: actor.birth_country_alpha2,
            name: actor.birth_country_name,
            alias: actor.birth_country_alias,
        };
    }

    if (actor.residence_city) curatedActor.residence.city = actor.residence_city;
    if (actor.residence_state) curatedActor.residence.state = actor.residence_state;

    if (actor.residence_country_alpha2) {
        curatedActor.residence.country = {
            alpha2: actor.residence_country_alpha2,
            name: actor.residence_country_name,
            alias: actor.residence_country_alias,
        };
    }

    return curatedActor;
}

function curateActors(releases) {
    return Promise.all(releases.map(async release => curateActor(release)));
}

function curateActorEntry(actor, scraped, scrapeSuccess) {
    const curatedActor = {
        name: actor.name
            .split(' ')
            .map(segment => `${segment.charAt(0).toUpperCase()}${segment.slice(1)}`)
            .join(' '),
        slug: slugify(actor.name),
        birthdate: actor.birthdate,
        description: actor.description,
        gender: actor.gender,
        ethnicity: actor.ethnicity,
        bust: actor.bust,
        waist: actor.waist,
        hip: actor.hip,
        natural_boobs: actor.naturalBoobs,
        height: actor.height,
        weight: actor.weight,
        hair: actor.hair,
        eyes: actor.eyes,
        has_tattoos: actor.hasTattoos,
        has_piercings: actor.hasPiercings,
        tattoos: actor.tattoos,
        piercings: actor.piercings,
    };

    if (actor.id) {
        curatedActor.id = actor.id;
    }

    if (actor.birthPlace) {
        curatedActor.birth_city = actor.birthPlace.city;
        curatedActor.birth_state = actor.birthPlace.state;
        curatedActor.birth_country_alpha2 = actor.birthPlace.country;
    }

    if (actor.residencePlace) {
        curatedActor.residence_city = actor.residencePlace.city;
        curatedActor.residence_state = actor.residencePlace.state;
        curatedActor.residence_country_alpha2 = actor.residencePlace.country;
    }

    if (scraped) {
        curatedActor.scraped_at = new Date();
        curatedActor.scrape_success = scrapeSuccess;
    }

    return curatedActor;
}

function curateSocialEntry(url, actorId) {
    const platforms = [
        // links supplied by PH often look like domain.com/domain.com/username
        {
            label: 'twitter',
            pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
            format: username => `https://www.twitter.com/${username}`,
        },
        {
            label: 'youtube',
            pattern: 'http(s)\\://(*)youtube.com/channel/:username(?*)',
            format: username => `https://www.youtube.com/channel/${username}`,
        },
        {
            label: 'instagram',
            pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
            format: username => `https://www.instagram.com/${username}`,
        },
        {
            label: 'snapchat',
            pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
            format: username => `https://www.snapchat.com/add/${username}`,
        },
        {
            label: 'tumblr',
            pattern: 'http(s)\\://:username.tumblr.com(*)',
            format: username => `https://${username}.tumblr.com`,
        },
        {
            label: 'onlyfans',
            pattern: 'http(s)\\://(*)onlyfans.com/:username(/)(?*)',
            format: username => `https://www.onlyfans.com/${username}`,
        },
        {
            label: 'fancentro',
            pattern: 'http(s)\\://(*)fancentro.com/:username(/)(?*)',
            format: username => `https://www.fancentro.com/${username}`,
        },
        {
            label: 'modelhub',
            pattern: 'http(s)\\://(*)modelhub.com/:username(/)(?*)',
            format: username => `https://www.modelhub.com/${username}`,
        },
    ];

    const match = platforms.reduce((acc, platform) => {
        if (acc) return acc;

        const patternMatch = new UrlPattern(platform.pattern).match(url);

        if (patternMatch) {
            return {
                platform: platform.label,
                original: url,
                username: patternMatch.username,
                url: platform.format ? platform.format(patternMatch.username) : url,
            };
        }

        return null;
    }, null) || { url };

    return {
        url: match.url,
        platform: match.platform,
        actor_id: actorId,
    };
}

async function curateSocialEntries(urls, actorId) {
    if (!urls) {
        return [];
    }

    const existingSocialLinks = await knex('actors_social').where('actor_id', actorId);

    return urls.reduce((acc, url) => {
        const socialEntry = curateSocialEntry(url, actorId);

        if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
            // prevent duplicates
            return acc;
        }

        return [...acc, socialEntry];
    }, []);
}

async function fetchActors(queryObject, limit = 100) {
    const releases = await knex('actors')
        .select(
            'actors.*',
            'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
            'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
        )
        .leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
        .leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
        .orderBy(['actors.name', 'actors.gender'])
        .where(builder => whereOr(queryObject, 'actors', builder))
        .limit(limit);

    return curateActors(releases);
}

async function storeSocialLinks(urls, actorId) {
    const curatedSocialEntries = await curateSocialEntries(urls, actorId);

    await knex('actors_social').insert(curatedSocialEntries);
}

async function storeActor(actor, scraped = false, scrapeSuccess = false) {
    const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);

    const [actorEntry] = await knex('actors')
        .insert(curatedActor)
        .returning('*');

    await storeSocialLinks(actor.social, actorEntry.id);

    if (actor.avatars) {
        await createMediaDirectory('actors', `${actorEntry.slug}/`);
        await storePhotos(actor.avatars, {
            domain: 'actor',
            role: 'photo',
            primaryRole: 'avatar',
            targetId: actorEntry.id,
            subpath: `${actorEntry.slug}/`,
            naming: 'timestamp',
        }, actorEntry.name);
    }

    logger.info(`Added new entry for actor '${actor.name}'`);

    return actorEntry;
}

async function updateActor(actor, scraped = false, scrapeSuccess = false) {
    const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);

    const [actorEntry] = await knex('actors')
        .where({ id: actor.id })
        .update(curatedActor)
        .returning('*');

    await storeSocialLinks(actor.social, actor.id);

    logger.info(`Updated entry for actor '${actor.name}'`);

    return actorEntry;
}

async function mergeProfiles(profiles, actor) {
    if (profiles.filter(Boolean).length === 0) {
        return null;
    }

    const mergedProfile = profiles.reduce((prevProfile, profile) => {
        if (profile === null) {
            return prevProfile;
        }

        return {
            id: actor ? actor.id : null,
            name: actor ? actor.name : (prevProfile.name || profile.name),
            description: prevProfile.description || profile.description,
            gender: prevProfile.gender || profile.gender,
            birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
            birthPlace: prevProfile.birthPlace || profile.birthPlace,
            residencePlace: prevProfile.residencePlace || profile.residencePlace,
            nationality: prevProfile.nationality || profile.nationality, // used to derive country when not available
            ethnicity: prevProfile.ethnicity || profile.ethnicity,
            bust: prevProfile.bust || profile.bust,
            waist: prevProfile.waist || profile.waist,
            hip: prevProfile.hip || profile.hip,
            naturalBoobs: prevProfile.naturalBoobs === undefined ? profile.naturalBoobs : prevProfile.naturalBoobs,
            height: prevProfile.height || profile.height,
            weight: prevProfile.weight || profile.weight,
            hair: prevProfile.hair || profile.hair,
            eyes: prevProfile.eyes || profile.eyes,
            hasPiercings: prevProfile.hasPiercings === undefined ? profile.hasPiercings : prevProfile.hasPiercings,
            hasTattoos: prevProfile.hasTattoos === undefined ? profile.hasTattoos : prevProfile.hasTattoos,
            piercings: prevProfile.piercings || profile.piercings,
            tattoos: prevProfile.tattoos || profile.tattoos,
            social: prevProfile.social.concat(profile.social || []),
            avatars: prevProfile.avatars.concat(profile.avatar ? [profile.avatar] : []), // don't flatten fallbacks
            releases: prevProfile.releases.concat(profile.releases ? profile.releases : []), // don't flatten fallbacks
        };
    }, {
        social: [],
        avatars: [],
        releases: [],
    });

    const [birthPlace, residencePlace] = await Promise.all([
        resolvePlace(mergedProfile.birthPlace),
        resolvePlace(mergedProfile.residencePlace),
    ]);

    mergedProfile.birthPlace = birthPlace;
    mergedProfile.residencePlace = residencePlace;

    if (!mergedProfile.birthPlace && mergedProfile.nationality) {
        const country = await knex('countries')
            .where('nationality', 'ilike', `%${mergedProfile.nationality}%`)
            .orderBy('priority', 'desc')
            .first();

        mergedProfile.birthPlace = {
            country: country.alpha2,
        };
    }

    return mergedProfile;
}

async function scrapeActors(actorNames) {
    return Promise.map(actorNames || argv.actors, async (actorName) => {
        try {
            const actorSlug = slugify(actorName);
            const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
            const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);

            const finalSources = argv.withReleases ? sources.flat() : sources; // ignore race-to-success grouping when scenes are requested

            const profiles = await Promise.map(finalSources, async (source) => {
               //  const [scraperSlug, scraper] = source;
                const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));

                try {
                    return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
                        if (!scraper) {
                            logger.warn(`No profile profile scraper available for ${scraperSlug}`);
                            throw Object.assign(new Error(`No profile scraper available for ${scraperSlug}`));
                        }

                        logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);

                        const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);

                        if (profile) {
                            logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);

                            return {
                                ...profile,
                                name: actorName,
                                scraper: scraperSlug,
                            };
                        }

                        logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}`);
                        throw Object.assign(new Error(`Profile for ${actorName} not available on ${scraperSlug}`), { warn: false });
                    }), Promise.reject(new Error()));
                } catch (error) {
                    if (error.warn !== false) {
                        logger.warn(`Error in scraper ${source}: ${error.message}`);
                        // logger.error(error.stack);
                    }
                }

                return null;
            });

            const profile = await mergeProfiles(profiles, actorEntry);

            if (argv.inspect) {
                console.log(profile);
                logger.info(`Found ${profile.releases.length} releases for ${actorName}`);
            }

            if (profile === null) {
                logger.warn(`Could not find profile for actor '${actorName}'`);

                if (argv.save && !actorEntry) {
                    await storeActor({ name: actorName }, false, false);
                }

                return null;
            }

            if (argv.save) {
                if (actorEntry && profile) {
                    await createMediaDirectory('actors', `${actorEntry.slug}/`);

                    await Promise.all([
                        updateActor(profile, true, true),
                        // storeAvatars(profile, actorEntry),
                        storePhotos(profile.avatars, {
                            domain: 'actor',
                            role: 'photo',
                            primaryRole: 'avatar',
                            targetId: actorEntry.id,
                            subpath: `${actorEntry.slug}/`,
                            naming: 'timestamp',
                        }, actorEntry.name),
                    ]);

                    return profile;
                }

                await storeActor(profile, true, true);
            }

            return profile;
        } catch (error) {
            console.log(error);
            logger.warn(`${actorName}: ${error}`);

            return null;
        }
    }, {
        concurrency: 3,
    });
}

async function scrapeBasicActors() {
    const basicActors = await knex('actors').where('scraped_at', null);

    return scrapeActors(basicActors.map(actor => actor.name));
}

async function associateActors(mappedActors, releases) {
    const [existingActorEntries, existingAssociationEntries] = await Promise.all([
        knex('actors')
            .whereIn('name', Object.values(mappedActors).map(actor => actor.name))
            .orWhereIn('slug', Object.keys(mappedActors)),
        knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
    ]);

    const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
        try {
            const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
                || await storeActor(actor);

            // if a scene
            return Array.from(actor.releaseIds)
                .map(releaseId => ({
                    release_id: releaseId,
                    actor_id: actorEntry.id,
                }))
                .filter(association => !existingAssociationEntries
                    // remove associations already in database
                    .some(associationEntry => associationEntry.actor_id === association.actor_id
                        && associationEntry.release_id === association.release_id));
        } catch (error) {
            logger.error(actor.name, error);
            return null;
        }
    });

    await knex('releases_actors').insert(associations.filter(association => association).flat());

    // basic actor scraping is failure prone, don't run together with actor association
    // await scrapebasicactors(),
}

module.exports = {
    associateActors,
    fetchActors,
    scrapeActors,
    scrapeBasicActors,
};