traxxx/src/scrapers/mindgeek.js

'use strict';

/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');

const { ex } = require('../utils/q');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');

function getThumbs(scene) {
    if (scene.images.poster) {
        return scene.images.poster.map(image => image.xl.url);
    }

    if (scene.images.card_main_rect) {
        return scene.images.card_main_rect
            .concat(scene.images.card_secondary_rect || [])
            .map(image => image.xl.url.replace('.thumb', ''));
    }

    return [];
}

function scrapeLatestX(data, site) {
    const { id: entryId, title, description } = data;
    const hostname = site.parameters?.native ? site.url : site.network.url;
    const url = `${hostname}/scene/${entryId}/`;
    const date = new Date(data.dateReleased);
    const actors = data.actors.map(actor => actor.name);

    const tags = data.tags.map(tag => tag.name);

    const [poster, ...photos] = getThumbs(data);
    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
    const duration = data.videos.mediabook && data.videos.mediabook.length;

    return {
        url,
        entryId,
        title,
        description,
        actors,
        tags,
        duration,
        poster,
        photos,
        trailer: trailer && {
            src: trailer.urls.view,
            quality: parseInt(trailer.format, 10),
        },
        date,
        site,
    };
}

async function scrapeLatest(items, site) {
    return Promise.all(items.map(async data => scrapeLatestX(data, site)));
}

async function scrapeScene(data, url, site) {
    const { id: entryId, title, description } = data;
    const date = new Date(data.dateReleased);
    const actors = data.actors.map(actor => actor.name);

    const tags = data.tags.map(tag => tag.name);

    const [poster, ...photos] = getThumbs(data);
    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);

    const siteName = data.collections[0].name;
    const channel = siteName.replace(/\s+/g, '').toLowerCase();

    return {
        url,
        entryId,
        title,
        description,
        actors,
        tags,
        poster,
        photos,
        trailer: trailer && {
            src: trailer.urls.view,
            quality: parseInt(trailer.format, 10),
        },
        date,
        site,
        channel,
    };
}

function getUrl(site) {
    const { search } = new URL(site.url);

    if (search.match(/\?site=\d+/)) {
        return site.url;
    }

    if (site.parameters?.native) {
        return `${site.url}/scenes`;
    }


    if (site.parameters?.siteId) {
        return `${site.network.url}/scenes?site=${site.parameters.siteId}`;
    }

    throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
}

async function getSession(url) {
    const cookieJar = new CookieJar();
    const session = bhttp.session({ cookieJar });

    await session.get(url);

    const cookieString = await cookieJar.getCookieStringAsync(url);
    const { instance_token: instanceToken } = cookieToData(cookieString);

    return { session, instanceToken };
}

function scrapeProfile(data, html) {
    const { qa, qd } = ex(html);

    const profile = {
        gender: data.gender,
        description: data.bio,
        aliases: data.aliases,
    };

    const [bust, waist, hip] = data.measurements.split('-');

    if (bust) profile.bust = bust.toUpperCase();
    if (waist) profile.waist = waist;
    if (hip) profile.hip = hip;

    if (data.birthPlace) profile.birthPlace = data.birthPlace;
    if (data.height) profile.height = inchesToCm(data.height);
    if (data.weight) profile.weight = lbsToKg(data.weight);

    if (data.images.card_main_rect && data.images.card_main_rect[0]) {
        profile.avatar = data.images.card_main_rect[0].xl?.url
            || data.images.card_main_rect[0].lg?.url
            || data.images.card_main_rect[0].md?.url
            || data.images.card_main_rect[0].sm?.url
            || data.images.card_main_rect[0].xs?.url;
    }

    const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
    if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');

    return profile;
}

async function fetchLatest(site, page = 1) {
    const url = getUrl(site);
    const { search } = new URL(url);
    const siteId = new URLSearchParams(search).get('site');

    const { session, instanceToken } = await getSession(url);

    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
    const limit = 10;
    const apiUrl = site.parameters?.native
        ? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
        : `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;

    const res = await session.get(apiUrl, {
        headers: {
            Instance: instanceToken,
            Origin: site.url,
            Referer: url,
        },
    });

    if (res.statusCode === 200 && res.body.result) {
        return scrapeLatest(res.body.result, site);
    }

    return null;
}

async function fetchScene(url, site) {
    const entryId = url.match(/\d+/)[0];
    const { session, instanceToken } = await getSession(url);

    const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
        headers: {
            Instance: instanceToken,
        },
    });

    if (res.statusCode === 200 && res.body.result) {
        return scrapeScene(res.body.result, url, site);
    }

    return null;
}

async function fetchProfile(actorName, networkName, actorPath = 'model') {
    const url = `https://www.${networkName}.com`;
    const { session, instanceToken } = await getSession(url);

    const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
        headers: {
            Instance: instanceToken,
        },
    });

    if (res.statusCode === 200) {
        const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());

        if (actorData) {
            const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);

            if (actorRes.statusCode === 200) {
                return scrapeProfile(actorData, actorRes.body.toString());
            }
        }
    }

    return null;
}

module.exports = {
    scrapeLatestX,
    fetchLatest,
    fetchScene,
    fetchProfile,
};