traxxx/src/scrapers/mindgeek.js

237 lines
6.8 KiB
JavaScript
Raw Normal View History

'use strict';
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const moment = require('moment');
const { ex } = require('../utils/q');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');
function getThumbs(scene) {
if (scene.images.poster) {
return scene.images.poster.map(image => image.xl.url);
}
if (scene.images.card_main_rect) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map(image => image.xl.url.replace('.thumb', ''));
}
return [];
}
function scrapeLatestX(data, site) {
const { id: entryId, title, description } = data;
const hostname = site.parameters?.native ? site.url : site.network.url;
const url = `${hostname}/scene/${entryId}/`;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const tags = data.tags.map(tag => tag.name);
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
const duration = data.videos.mediabook && data.videos.mediabook.length;
return {
url,
entryId,
title,
description,
actors,
tags,
duration,
poster,
photos,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
date,
site,
};
}
async function scrapeLatest(items, site) {
return Promise.all(items.map(async data => scrapeLatestX(data, site)));
}
async function scrapeScene(data, url, site) {
const { id: entryId, title, description } = data;
const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name);
const tags = data.tags.map(tag => tag.name);
const [poster, ...photos] = getThumbs(data);
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
const siteName = data.collections[0].name;
const channel = siteName.replace(/\s+/g, '').toLowerCase();
return {
url,
entryId,
title,
description,
actors,
tags,
poster,
photos,
trailer: trailer && {
src: trailer.urls.view,
quality: parseInt(trailer.format, 10),
},
date,
site,
channel,
};
}
function getUrl(site) {
const { search } = new URL(site.url);
if (search.match(/\?site=\d+/)) {
return site.url;
}
if (site.parameters?.native) {
return `${site.url}/scenes`;
}
if (site.parameters?.siteId) {
return `${site.network.url}/scenes?site=${site.parameters.siteId}`;
}
throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
}
async function getSession(url) {
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
return { session, instanceToken };
}
function scrapeProfile(data, html) {
const { qa, qd } = ex(html);
const profile = {
description: data.bio,
aliases: data.aliases,
};
const [bust, waist, hip] = data.measurements.split('-');
profile.gender = data.gender === 'other' ? 'transsexual' : data.gender;
if (bust) profile.bust = bust.toUpperCase();
if (waist) profile.waist = waist;
if (hip) profile.hip = hip;
if (data.birthPlace) profile.birthPlace = data.birthPlace;
if (data.height) profile.height = inchesToCm(data.height);
if (data.weight) profile.weight = lbsToKg(data.weight);
2020-01-29 01:31:55 +00:00
if (data.images.card_main_rect && data.images.card_main_rect[0]) {
profile.avatar = data.images.card_main_rect[0].xl?.url
|| data.images.card_main_rect[0].lg?.url
|| data.images.card_main_rect[0].md?.url
|| data.images.card_main_rect[0].sm?.url
|| data.images.card_main_rect[0].xs?.url;
}
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
return profile;
}
async function fetchLatest(site, page = 1) {
const url = getUrl(site);
const { search } = new URL(url);
const siteId = new URLSearchParams(search).get('site');
const { session, instanceToken } = await getSession(url);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 10;
const apiUrl = site.parameters?.native
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
: `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
const res = await session.get(apiUrl, {
headers: {
Instance: instanceToken,
Origin: site.url,
Referer: url,
},
});
if (res.statusCode === 200 && res.body.result) {
return scrapeLatest(res.body.result, site);
}
return null;
}
async function fetchScene(url, site) {
const entryId = url.match(/\d+/)[0];
const { session, instanceToken } = await getSession(url);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
Instance: instanceToken,
},
});
if (res.statusCode === 200 && res.body.result) {
return scrapeScene(res.body.result, url, site);
}
return null;
}
async function fetchProfile(actorName, networkName, actorPath = 'model') {
const url = `https://www.${networkName}.com`;
const { session, instanceToken } = await getSession(url);
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
headers: {
Instance: instanceToken,
},
});
if (res.statusCode === 200) {
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
if (actorData) {
const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
if (actorRes.statusCode === 200) {
return scrapeProfile(actorData, actorRes.body.toString());
}
}
}
return null;
}
module.exports = {
scrapeLatestX,
fetchLatest,
fetchScene,
fetchProfile,
};