traxxx/src/scrapers/mindgeek.js

'use strict';

/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');

const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');

function getThumbs(scene) {
	if (scene.images.poster) {
		return scene.images.poster.map(image => image.xl.url);
	}

	if (scene.images.card_main_rect) {
		return scene.images.card_main_rect
			.concat(scene.images.card_secondary_rect || [])
			.map(image => image.xl.url.replace('.thumb', ''));
	}

	return [];
}

function scrapeLatestX(data, site) {
	if (site.parameters?.extract === true && data.collections.length > 0) {
		// release should not belong to any channel
		return null;
	}

	if (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) {
		// release should belong to specific channel
		return null;
	}

	const release = {
		entryId: data.id,
		title: data.title,
		description: data.description,
	};

	const hostname = site.parameters?.native ? site.url : site.parent.url;

	release.url = `${hostname}/scene/${release.entryId}/`;
	release.date = new Date(data.dateReleased);
	release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));

	release.tags = data.tags.map(tag => tag.name);

	release.duration = data.videos.mediabook?.length;
	[release.poster, ...release.photos] = getThumbs(data);

	const teaserSources = data.videos.mediabook?.files;

	if (teaserSources) {
		release.teaser = Object.values(teaserSources).map(teaser => ({
			src: teaser.urls.view,
			quality: parseInt(teaser.format, 10),
		}));
	}

	return release;
}

async function scrapeLatest(items, site) {
	const latestReleases = await Promise.all(items.map(async data => scrapeLatestX(data, site)));

	return latestReleases.filter(Boolean);
}

function scrapeScene(data, url, _site, networkName) {
	const release = {};

	const { id: entryId, title, description } = data;

	release.entryId = data.id;
	release.title = title;
	release.description = description;

	release.date = new Date(data.dateReleased);
	release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));

	release.tags = data.tags.map(tag => tag.name);

	[release.poster, ...release.photos] = getThumbs(data);

	const teaserSources = data.videos.mediabook?.files;

	if (teaserSources) {
		release.teaser = Object.values(teaserSources).map(teaser => ({
			src: teaser.urls.view,
			quality: parseInt(teaser.format, 10),
		}));
	}

	const siteName = data.collections[0]?.name || data.brand;
	release.channel = slugify(siteName, '');

	release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`;

	return release;
}

function getUrl(site) {
	const { search } = new URL(site.url);

	if (search.match(/\?site=\d+/)) {
		return site.url;
	}

	if (site.parameters?.native) {
		return `${site.url}/scenes`;
	}

	if (site.parameters?.extract) {
		return `${site.url}/scenes`;
	}

	if (site.parameters?.siteId) {
		return `${site.parent.url}/scenes?site=${site.parameters.siteId}`;
	}

	throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
}

async function getSession(url) {
	const cookieJar = new CookieJar();
	const session = bhttp.session({ cookieJar });

	await session.get(url);

	const cookieString = await cookieJar.getCookieStringAsync(url);
	const { instance_token: instanceToken } = cookieToData(cookieString);

	return { session, instanceToken };
}

function scrapeProfile(data, html, releases = [], networkName) {
	const { query } = qu.extract(html);

	const profile = {
		description: data.bio,
		aliases: data.aliases,
	};

	const [bust, waist, hip] = data.measurements.split('-');

	profile.gender = data.gender === 'other' ? 'transsexual' : data.gender;

	if (profile.gender === 'female') {
		if (bust) profile.bust = bust.toUpperCase();
		if (waist) profile.waist = waist;
		if (hip) profile.hip = hip;
	}

	if (data.birthPlace) profile.birthPlace = data.birthPlace;
	if (data.height) profile.height = inchesToCm(data.height);
	if (data.weight) profile.weight = lbsToKg(data.weight);

	if (data.images.card_main_rect?.[0]) {
		profile.avatar = data.images.card_main_rect[0].xl?.url
            || data.images.card_main_rect[0].lg?.url
            || data.images.card_main_rect[0].md?.url
            || data.images.card_main_rect[0].sm?.url
            || data.images.card_main_rect[0].xs?.url;
	}

	const birthdate = query.all('li').find(el => /Date of Birth/.test(el.textContent));
	if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');

	profile.releases = releases.map(release => scrapeScene(release, null, null, networkName));

	return profile;
}

async function fetchLatest(site, page = 1) {
	const url = getUrl(site);
	const { search } = new URL(url);
	const siteId = new URLSearchParams(search).get('site');

	const { session, instanceToken } = await getSession(url);

	const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
	const limit = 10;
	const apiUrl = site.parameters?.native || site.parameters?.extract
		? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
		: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;

	const res = await session.get(apiUrl, {
		headers: {
			Instance: instanceToken,
			Origin: site.url,
			Referer: url,
		},
	});

	if (res.statusCode === 200 && res.body.result) {
		return scrapeLatest(res.body.result, site);
	}

	return null;
}

async function fetchScene(url, site) {
	const entryId = url.match(/\d+/)[0];
	const { session, instanceToken } = await getSession(url);

	const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
		headers: {
			Instance: instanceToken,
		},
	});

	if (res.statusCode === 200 && res.body.result) {
		return scrapeScene(res.body.result, url, site);
	}

	return null;
}

async function fetchProfile({ name: actorName }, networkSlug, actorPath = 'model') {
	const url = `https://www.${networkSlug}.com`;
	const { session, instanceToken } = await getSession(url);

	const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
		headers: {
			Instance: instanceToken,
		},
	});

	if (res.statusCode === 200) {
		const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());

		if (actorData) {
			const actorUrl = `https://www.${networkSlug}.com/${actorPath}/${actorData.id}/`;
			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;

			const [actorRes, actorReleasesRes] = await Promise.all([
				bhttp.get(actorUrl),
				session.get(actorReleasesUrl, {
					headers: {
						Instance: instanceToken,
					},
				}),
			]);

			if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
				return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkSlug);
			}

			if (actorRes.statusCode === 200) {
				return scrapeProfile(actorData, actorRes.body.toString(), null, networkSlug);
			}
		}
	}

	return null;
}

module.exports = {
	scrapeLatestX,
	fetchLatest,
	fetchScene,
	fetchProfile,
};