2020-01-14 03:50:42 +00:00
|
|
|
|
|
|
|
'use strict';
|
|
|
|
|
|
|
|
/* eslint-disable newline-per-chained-call */
|
|
|
|
const Promise = require('bluebird');
|
|
|
|
const bhttp = require('bhttp');
|
|
|
|
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
|
|
|
const moment = require('moment');
|
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
const { ex } = require('../utils/q');
|
|
|
|
const { inchesToCm, lbsToKg } = require('../utils/convert');
|
2020-01-14 03:50:42 +00:00
|
|
|
const { cookieToData } = require('../utils/cookies');
|
|
|
|
|
|
|
|
function getThumbs(scene) {
|
|
|
|
if (scene.images.poster) {
|
|
|
|
return scene.images.poster.map(image => image.xl.url);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (scene.images.card_main_rect) {
|
|
|
|
return scene.images.card_main_rect
|
|
|
|
.concat(scene.images.card_secondary_rect || [])
|
|
|
|
.map(image => image.xl.url.replace('.thumb', ''));
|
|
|
|
}
|
|
|
|
|
|
|
|
return [];
|
|
|
|
}
|
|
|
|
|
|
|
|
function scrapeLatestX(data, site) {
|
|
|
|
const { id: entryId, title, description } = data;
|
|
|
|
const hostname = site.parameters?.classic ? site.url : site.network.url;
|
|
|
|
const url = `${hostname}/scene/${entryId}/`;
|
|
|
|
const date = new Date(data.dateReleased);
|
|
|
|
const actors = data.actors.map(actor => actor.name);
|
|
|
|
|
|
|
|
const tags = data.tags.map(tag => tag.name);
|
|
|
|
|
|
|
|
const [poster, ...photos] = getThumbs(data);
|
|
|
|
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
|
|
|
|
const duration = data.videos.mediabook && data.videos.mediabook.length;
|
|
|
|
|
|
|
|
return {
|
|
|
|
url,
|
|
|
|
entryId,
|
|
|
|
title,
|
|
|
|
description,
|
|
|
|
actors,
|
|
|
|
tags,
|
|
|
|
duration,
|
|
|
|
poster,
|
|
|
|
photos,
|
|
|
|
trailer: trailer && {
|
|
|
|
src: trailer.urls.view,
|
|
|
|
quality: parseInt(trailer.format, 10),
|
|
|
|
},
|
|
|
|
date,
|
|
|
|
site,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
async function scrapeLatest(items, site) {
|
|
|
|
return Promise.all(items.map(async data => scrapeLatestX(data, site)));
|
|
|
|
}
|
|
|
|
|
|
|
|
async function scrapeScene(data, url, site) {
|
|
|
|
const { id: entryId, title, description } = data;
|
|
|
|
const date = new Date(data.dateReleased);
|
|
|
|
const actors = data.actors.map(actor => actor.name);
|
|
|
|
|
|
|
|
const tags = data.tags.map(tag => tag.name);
|
|
|
|
|
|
|
|
const [poster, ...photos] = getThumbs(data);
|
|
|
|
const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
|
|
|
|
|
|
|
|
const siteName = data.collections[0].name;
|
|
|
|
const channel = siteName.replace(/\s+/g, '').toLowerCase();
|
|
|
|
|
|
|
|
return {
|
|
|
|
url,
|
|
|
|
entryId,
|
|
|
|
title,
|
|
|
|
description,
|
|
|
|
actors,
|
|
|
|
tags,
|
|
|
|
poster,
|
|
|
|
photos,
|
|
|
|
trailer: trailer && {
|
|
|
|
src: trailer.urls.view,
|
|
|
|
quality: parseInt(trailer.format, 10),
|
|
|
|
},
|
|
|
|
date,
|
|
|
|
site,
|
|
|
|
channel,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
function getUrl(site) {
|
|
|
|
const { search } = new URL(site.url);
|
|
|
|
|
|
|
|
if (search.match(/\?site=\d+/)) {
|
|
|
|
return site.url;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (site.parameters && site.parameters.siteId) {
|
|
|
|
return `${site.network.url}/scenes?site=${site.parameters.siteId}`;
|
|
|
|
}
|
|
|
|
|
|
|
|
throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`);
|
|
|
|
}
|
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
async function getSession(url) {
|
2020-01-14 03:50:42 +00:00
|
|
|
const cookieJar = new CookieJar();
|
|
|
|
const session = bhttp.session({ cookieJar });
|
|
|
|
|
|
|
|
await session.get(url);
|
|
|
|
|
|
|
|
const cookieString = await cookieJar.getCookieStringAsync(url);
|
|
|
|
const { instance_token: instanceToken } = cookieToData(cookieString);
|
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
return { session, instanceToken };
|
|
|
|
}
|
|
|
|
|
|
|
|
function scrapeProfile(data, html) {
|
|
|
|
const { qa, qd } = ex(html);
|
|
|
|
|
|
|
|
const profile = {
|
|
|
|
gender: data.gender,
|
|
|
|
description: data.bio,
|
|
|
|
aliases: data.aliases,
|
|
|
|
};
|
|
|
|
|
|
|
|
const [bust, waist, hip] = data.measurements.split('-');
|
|
|
|
|
|
|
|
if (bust) profile.bust = bust.toUpperCase();
|
|
|
|
if (waist) profile.waist = waist;
|
|
|
|
if (hip) profile.hip = hip;
|
|
|
|
|
|
|
|
if (data.birthPlace) profile.birthPlace = data.birthPlace;
|
|
|
|
if (data.height) profile.height = inchesToCm(data.height);
|
|
|
|
if (data.weight) profile.weight = lbsToKg(data.weight);
|
|
|
|
|
2020-01-29 01:31:55 +00:00
|
|
|
if (data.images.card_main_rect && data.images.card_main_rect[0]) {
|
|
|
|
profile.avatar = data.images.card_main_rect[0].xl?.url
|
|
|
|
|| data.images.card_main_rect[0].lg?.url
|
|
|
|
|| data.images.card_main_rect[0].md?.url
|
|
|
|
|| data.images.card_main_rect[0].sm?.url
|
|
|
|
|| data.images.card_main_rect[0].xs?.url;
|
|
|
|
}
|
2020-01-27 21:54:14 +00:00
|
|
|
|
|
|
|
const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent));
|
|
|
|
if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY');
|
|
|
|
|
|
|
|
return profile;
|
|
|
|
}
|
|
|
|
|
|
|
|
async function fetchLatest(site, page = 1) {
|
|
|
|
const url = getUrl(site);
|
|
|
|
const { search } = new URL(url);
|
|
|
|
const siteId = new URLSearchParams(search).get('site');
|
|
|
|
|
|
|
|
const { session, instanceToken } = await getSession(url);
|
|
|
|
|
2020-01-14 03:50:42 +00:00
|
|
|
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
|
|
|
|
const limit = 10;
|
|
|
|
const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
|
|
|
|
const res = await session.get(apiUrl, {
|
|
|
|
headers: {
|
|
|
|
Instance: instanceToken,
|
|
|
|
},
|
|
|
|
});
|
|
|
|
|
|
|
|
return scrapeLatest(res.body.result, site);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function fetchScene(url, site) {
|
|
|
|
const entryId = url.match(/\d+/)[0];
|
2020-01-27 21:54:14 +00:00
|
|
|
const { session, instanceToken } = await getSession(url);
|
2020-01-14 03:50:42 +00:00
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
|
|
|
|
headers: {
|
|
|
|
Instance: instanceToken,
|
|
|
|
},
|
|
|
|
});
|
2020-01-14 03:50:42 +00:00
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
return scrapeScene(res.body.result, url, site);
|
|
|
|
}
|
2020-01-14 03:50:42 +00:00
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
async function fetchProfile(actorName, networkName, actorPath = 'model') {
|
|
|
|
const url = `https://www.${networkName}.com`;
|
|
|
|
const { session, instanceToken } = await getSession(url);
|
2020-01-14 03:50:42 +00:00
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, {
|
2020-01-14 03:50:42 +00:00
|
|
|
headers: {
|
|
|
|
Instance: instanceToken,
|
|
|
|
},
|
|
|
|
});
|
|
|
|
|
2020-01-27 21:54:14 +00:00
|
|
|
if (res.statusCode === 200) {
|
|
|
|
const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase());
|
|
|
|
|
2020-01-28 02:05:53 +00:00
|
|
|
if (actorData) {
|
|
|
|
const actorRes = await bhttp.get(`https://www.${networkName}.com/${actorPath}/${actorData.id}/`);
|
|
|
|
|
|
|
|
if (actorRes.statusCode === 200) {
|
|
|
|
return scrapeProfile(actorData, actorRes.body.toString());
|
|
|
|
}
|
2020-01-27 21:54:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
2020-01-14 03:50:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
module.exports = {
|
|
|
|
scrapeLatestX,
|
|
|
|
fetchLatest,
|
|
|
|
fetchScene,
|
2020-01-27 21:54:14 +00:00
|
|
|
fetchProfile,
|
2020-01-14 03:50:42 +00:00
|
|
|
};
|