traxxx/src/scrapers/realitykings.js

'use strict';

/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const cheerio = require('cheerio');
const moment = require('moment');

const { cookieToData } = require('../utils/cookies');

function getThumbs(scene) {
    if (scene.images.poster) {
        return scene.images.poster.map(image => image.xl.url);
    }

    if (scene.images.card_main_rect) {
        return scene.images.card_main_rect
            .concat(scene.images.card_secondary_rect || [])
            .map(image => image.xl.url.replace('.thumb', ''));
    }

    return [];
}

async function scrapeLatest(items, site) {
    return Promise.all(items.map(async (data) => {
        const { id: entryId, title, description } = data;
        const url = `https://www.realitykings.com/scene/${entryId}/`;
        const date = new Date(data.dateReleased);
        const actors = data.actors.map(actor => actor.name);

        const tags = data.tags.map(tag => tag.name);

        const [poster, ...photos] = getThumbs(data);
        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
        const duration = data.videos.mediabook && data.videos.mediabook.length;

        return {
            url,
            entryId,
            title,
            description,
            actors,
            tags,
            duration,
            poster,
            photos,
            trailer: trailer && {
                src: trailer.urls.view,
                quality: parseInt(trailer.format, 10),
            },
            date,
            site,
        };
    }));
}

function scrapeLatestClassic(html, site) {
    const $ = cheerio.load(html, { normalizeWhitespace: true });

    const stateTag = $('script:contains("initialState")').html();
    const prefix = 'initialState = {';
    const prefixIndex = stateTag.indexOf('initialState = {');
    const suffix = '};';
    const stateString = stateTag.slice(prefixIndex + prefix.length - 1, stateTag.indexOf('};', prefixIndex) + suffix.length - 1);
    const data = JSON.parse(stateString);

    return Object.values(data.entities.releases).map((scene) => {
        const release = { site };

        release.entryId = scene.id;
        release.url = `${site.url}/scene/${release.entryId}/`;

        release.title = scene.title;
        release.description = scene.description;
        release.date = new Date(scene.dateReleased);

        release.actors = scene.actors.map(actorId => data.entities.actors[actorId].name);
        release.tags = scene.tags.map(tagId => data.entities.tags[tagId].name);

        const [poster, ...photos] = getThumbs(scene);

        release.poster = poster;
        release.photos = photos;

        if (scene.videos.mediabook) {
            const trailer = (scene.videos.mediabook.files['720p'] || scene.videos.mediabook.files['320p']);

            release.duration = scene.videos.mediabook.length;
            release.trailer = trailer && {
                src: trailer.urls.view,
                quality: parseInt(trailer.format, 10),
            };
        }

        return release;
    });
}

async function scrapeScene(data, url, site) {
    const { id: entryId, title, description } = data;
    const date = new Date(data.dateReleased);
    const actors = data.actors.map(actor => actor.name);

    const tags = data.tags.map(tag => tag.name);

    const [poster, ...photos] = getThumbs(data);
    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);

    const siteName = data.collections[0].name;
    // const siteId = data.collections[0].id;
    const channel = siteName.replace(/\s+/g, '').toLowerCase();
    // const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;

    return {
        url,
        entryId,
        title,
        description,
        actors,
        tags,
        poster,
        photos,
        trailer: trailer && {
            src: trailer.urls.view,
            quality: parseInt(trailer.format, 10),
        },
        date,
        site,
        channel,
    };
}

function getUrl(site) {
    const { hostname, search } = new URL(site.url);

    if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
        return site.url;
    }

    if (site.parameters && site.parameters.siteId) {
        return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
    }

    throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
}

async function fetchClassic(site, page) {
    const res = await bhttp.get(`${site.url}/scenes?page=${page}`);

    if (res.statusCode === 200) {
        return scrapeLatestClassic(res.body.toString(), site);
    }

    return null;
}

async function fetchLatest(site, page = 1) {
    if (site.parameters && site.parameters.classic) {
        return fetchClassic(site, page);
    }

    const url = getUrl(site);
    const { search } = new URL(url);
    const siteId = new URLSearchParams(search).get('site');

    const cookieJar = new CookieJar();
    const session = bhttp.session({ cookieJar });

    await session.get(url);

    const cookieString = await cookieJar.getCookieStringAsync(url);
    const { instance_token: instanceToken } = cookieToData(cookieString);

    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
    const limit = 10;
    const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
    const res = await session.get(apiUrl, {
        headers: {
            Instance: instanceToken,
        },
    });

    return scrapeLatest(res.body.result, site);
}

async function fetchScene(url, site) {
    const entryId = url.match(/\d+/)[0];

    const cookieJar = new CookieJar();
    const session = bhttp.session({ cookieJar });

    await session.get(url);

    const cookieString = await cookieJar.getCookieStringAsync(url);
    const { instance_token: instanceToken } = cookieToData(cookieString);

    const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
        headers: {
            Instance: instanceToken,
        },
    });

    return scrapeScene(res.body.result, url, site);
}

module.exports = {
    fetchLatest,
    fetchScene,
};