'use strict'; /* eslint-disable newline-per-chained-call */ const Promise = require('bluebird'); const bhttp = require('bhttp'); const { CookieJar } = Promise.promisifyAll(require('tough-cookie')); const moment = require('moment'); const { ex } = require('../utils/q'); const { inchesToCm, lbsToKg } = require('../utils/convert'); const { cookieToData } = require('../utils/cookies'); function getThumbs(scene) { if (scene.images.poster) { return scene.images.poster.map(image => image.xl.url); } if (scene.images.card_main_rect) { return scene.images.card_main_rect .concat(scene.images.card_secondary_rect || []) .map(image => image.xl.url.replace('.thumb', '')); } return []; } function scrapeLatestX(data, site) { if (site.parameters?.extract === true && data.collections.length > 0) { // release should not belong to any channel return null; } if (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) { // release should belong to specific channel return null; } const { id: entryId, title, description } = data; const hostname = site.parameters?.native ? site.url : site.network.url; const url = `${hostname}/scene/${entryId}/`; const date = new Date(data.dateReleased); const actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender })); const tags = data.tags.map(tag => tag.name); const [poster, ...photos] = getThumbs(data); const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']); const duration = data.videos.mediabook && data.videos.mediabook.length; return { url, entryId, title, description, actors, tags, duration, poster, photos, trailer: trailer && { src: trailer.urls.view, quality: parseInt(trailer.format, 10), }, date, site, }; } async function scrapeLatest(items, site) { const latestReleases = await Promise.all(items.map(async data => scrapeLatestX(data, site))); return latestReleases.filter(Boolean); } function scrapeScene(data, url, _site, networkName) { const release = {}; const { id: entryId, title, description } = data; release.entryId = data.id; release.title = title; release.description = description; release.date = new Date(data.dateReleased); release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender })); release.tags = data.tags.map(tag => tag.name); [release.poster, ...release.photos] = getThumbs(data); const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']); if (trailer) { release.trailer = { src: trailer.urls.view, quality: parseInt(trailer.format, 10), }; } const siteName = data.collections[0]?.name || data.brand; release.channel = siteName.replace(/\s+/g, '').toLowerCase(); release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`; return release; } function getUrl(site) { const { search } = new URL(site.url); if (search.match(/\?site=\d+/)) { return site.url; } if (site.parameters?.native) { return `${site.url}/scenes`; } if (site.parameters?.extract) { return `${site.url}/scenes`; } if (site.parameters?.siteId) { return `${site.network.url}/scenes?site=${site.parameters.siteId}`; } throw new Error(`Mind Geek site '${site.name}' (${site.url}) not supported`); } async function getSession(url) { const cookieJar = new CookieJar(); const session = bhttp.session({ cookieJar }); await session.get(url); const cookieString = await cookieJar.getCookieStringAsync(url); const { instance_token: instanceToken } = cookieToData(cookieString); return { session, instanceToken }; } function scrapeProfile(data, html, releases = [], networkName) { const { qa, qd } = ex(html); const profile = { description: data.bio, aliases: data.aliases, }; const [bust, waist, hip] = data.measurements.split('-'); profile.gender = data.gender === 'other' ? 'transsexual' : data.gender; if (bust) profile.bust = bust.toUpperCase(); if (waist) profile.waist = waist; if (hip) profile.hip = hip; if (data.birthPlace) profile.birthPlace = data.birthPlace; if (data.height) profile.height = inchesToCm(data.height); if (data.weight) profile.weight = lbsToKg(data.weight); if (data.images.card_main_rect?.[0]) { profile.avatar = data.images.card_main_rect[0].xl?.url || data.images.card_main_rect[0].lg?.url || data.images.card_main_rect[0].md?.url || data.images.card_main_rect[0].sm?.url || data.images.card_main_rect[0].xs?.url; } const birthdate = qa('li').find(el => /Date of Birth/.test(el.textContent)); if (birthdate) profile.birthdate = qd(birthdate, 'span', 'MMMM Do, YYYY'); profile.releases = releases.map(release => scrapeScene(release, null, null, networkName)); return profile; } async function fetchLatest(site, page = 1) { const url = getUrl(site); const { search } = new URL(url); const siteId = new URLSearchParams(search).get('site'); const { session, instanceToken } = await getSession(url); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const limit = 10; const apiUrl = site.parameters?.native || site.parameters?.extract ? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene` : `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`; const res = await session.get(apiUrl, { headers: { Instance: instanceToken, Origin: site.url, Referer: url, }, }); if (res.statusCode === 200 && res.body.result) { return scrapeLatest(res.body.result, site); } return null; } async function fetchScene(url, site) { const entryId = url.match(/\d+/)[0]; const { session, instanceToken } = await getSession(url); const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, { headers: { Instance: instanceToken, }, }); if (res.statusCode === 200 && res.body.result) { return scrapeScene(res.body.result, url, site); } return null; } async function fetchProfile(actorName, networkName, actorPath = 'model') { const url = `https://www.${networkName}.com`; const { session, instanceToken } = await getSession(url); const res = await session.get(`https://site-api.project1service.com/v1/actors/?search=${encodeURI(actorName)}`, { headers: { Instance: instanceToken, }, }); if (res.statusCode === 200) { const actorData = res.body.result.find(actor => actor.name.toLowerCase() === actorName.toLowerCase()); if (actorData) { const actorUrl = `https://www.${networkName}.com/${actorPath}/${actorData.id}/`; const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`; const [actorRes, actorReleasesRes] = await Promise.all([ bhttp.get(actorUrl), session.get(actorReleasesUrl, { headers: { Instance: instanceToken, }, }), ]); if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) { return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, networkName); } if (actorRes.statusCode === 200) { return scrapeProfile(actorData, actorRes.body.toString(), null, networkName); } } } return null; } module.exports = { scrapeLatestX, fetchLatest, fetchScene, fetchProfile, };