'use strict'; const Promise = require('bluebird'); const logger = require('../logger'); const { fetchApiLatest } = require('./gamma'); const qu = require('../utils/qu'); const http = require('../utils/http'); const slugify = require('../utils/slugify'); async function fetchActors(entryId, channel, { token, time }) { const url = `${channel.url}/sapi/${token}/${time}/model.getModelContent?_method=model.getModelContent&tz=1&fields[0]=modelId.stageName&fields[1]=_last&fields[2]=modelId.upsellLink&fields[3]=modelId.upsellText&limit=25&transitParameters[contentId]=${entryId}`; const res = await http.get(url); if (res.statusCode === 200 && res.body.status === true) { return Object.values(res.body.response.collection).map(actor => Object.values(actor.modelId.collection)[0].stageName); } return []; } async function fetchTrailerLocation(entryId, channel) { const url = `${channel.url}/api/download/${entryId}/hd1080/stream`; try { const res = await http.get(url, null, { followRedirects: false, }); if (res.statusCode === 302) { return res.headers.location; } } catch (error) { logger.warn(`${channel.name}: Unable to fetch trailer at '${url}': ${error.message}`); } return null; } function scrapeLatest(items, channel) { return items.map(({ query }) => { const release = {}; release.url = query.url('h5 a', null, { origin: channel.url }); release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1]; release.title = query.cnt('h5 a'); [release.poster, ...release.photos] = query.imgs('.screenshot').map(src => [ // unnecessarily large // src.replace(/\/\d+/, 3840), // src.replace(/\/\d+/, '/2000'), src.replace(/\/\d+/, '/1500'), src.replace(/\/\d+/, '/1000'), src, ]); return release; }); } function scrapeScene({ query, html }, url, channel) { const release = {}; release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1]; release.title = query.cnt('h1.description'); release.actors = query .all('.video-performer') .map((actorEl) => { const actorUrl = query.url(actorEl, 'a', 'href', { origin: channel.url }); const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1]; const avatar = query.img(actorEl, 'img:not([data-bgsrc*="not-available"])', 'data-bgsrc'); return { name: query.cnt(actorEl, '.video-performer-name'), gender: 'female', avatar: avatar && [ avatar.replace(/\/actor\/(\d+)/, '/actor/500'), avatar, ], url: actorUrl, entryId, }; }) .concat({ name: 'Jay Rock', gender: 'male' }); release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); release.duration = query.number('.release-date:last-child') * 60; release.studio = query.cnt('.studio span:nth-child(2)'); release.director = query.text('.director'); release.tags = query.cnts('.tags a'); const poster = html.match(/url\((https.+\.jpg)\)/)?.[1]; const photos = query.imgs('#moreScreenshots img'); [release.poster, ...release.photos] = [poster] .concat(photos) .filter(Boolean) .map(src => [ src.replace(/\/(\d+)\/\d+/, '/$1/1500'), src.replace(/\/(\d+)\/\d+/, '/$1/1000'), src, ]); const videoId = html.match(/item: (\d+)/)?.[1]; if (videoId) { release.trailer = { stream: `https://trailer.adultempire.com/hls/trailer/${videoId}/master.m3u8` }; } return release; } async function scrapeSceneApi(scene, channel, tokens, deep) { const release = { entryId: scene.id, title: scene.title, duration: scene.length, meta: { tokens, // attach tokens to reduce number of requests required for deep fetching }, }; release.url = `${channel.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`; release.date = new Date(scene.sites.collection[scene.id].publishDate); release.poster = scene._resources.primary[0].url; if (scene.tags) release.tags = Object.values(scene.tags.collection).map(tag => tag.alias); if (scene._resources.base) release.photos = scene._resources.base.map(resource => resource.url); if (deep) { // don't make external requests during update scraping, as this would happen for every scene on the page const [actors, trailer] = await Promise.all([ fetchActors(release.entryId, channel, tokens), fetchTrailerLocation(release.entryId, channel), ]); release.actors = actors; if (trailer) { release.trailer = { src: trailer, quality: 1080 }; } } return release; } function scrapeLatestApi(scenes, site, tokens) { return Promise.map(scenes, async scene => scrapeSceneApi(scene, site, tokens, false), { concurrency: 10 }); } async function fetchToken(channel) { const res = await http.get(channel.url); const html = res.body.toString(); const time = html.match(/"aet":\d+/)[0].split(':')[1]; const ah = html.match(/"ah":"[\w-]+"/)[0].split(':')[1].slice(1, -1); const token = ah.split('').reverse().join(''); return { time, token }; } async function fetchLatestApi(channel, page = 1) { const { time, token } = await fetchToken(channel); // transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory const url = `${channel.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`; const res = await http.get(url); if (res.ok && res.body.status) { return scrapeLatestApi(res.body.response.collection, channel, { time, token }); } return res.ok ? res.body.status : res.status; } async function fetchLatest(channel, page = 1, options, preData) { if (channel.parameters?.useApi) { return fetchLatestApi(channel, page, options, preData); } if (channel.parameters?.useGamma) { return fetchApiLatest(channel, page, preData, options, false); } const res = await qu.getAll(`https://jayspov.net/jays-pov-updates.html?view=list&page=${page}`, '.item-grid-list-view > .grid-item'); if (res.ok) { return scrapeLatest(res.items, channel); } return res.status; } async function fetchSceneApi(url, channel, baseRelease) { const { time, token } = baseRelease?.meta.tokens || await fetchToken(channel); // use attached tokens when deep fetching const { pathname } = new URL(url); const entryId = pathname.split('/')[2]; const apiUrl = `${channel.url}/sapi/${token}/${time}/content.load?filter[id][fields][0]=id&filter[id][values][0]=${entryId}&transitParameters[v1]=ykYa8ALmUD&transitParameters[preset]=scene`; const res = await http.get(apiUrl); if (res.ok && res.body.status) { return scrapeSceneApi(res.body.response.collection[0], channel, { time, token }, true); } return res.ok ? res.body.status : res.status; } async function fetchScene(url, channel) { if (channel.parameters?.useApi) { return fetchSceneApi(url, channel); } const res = await qu.get(url); if (res.ok) { return scrapeScene(res.item, url, channel); } return res.status; } module.exports = { fetchLatest, fetchScene, };