'use strict'; const unprint = require('unprint'); const http = require('../utils/http'); const slugify = require('../utils/slugify'); const { convert } = require('../utils/convert'); function getChannelSlug(channelName, entity) { if (!channelName) { return null; } const channelSlug = slugify(channelName, '', { removePunctuation: true }); if (entity.type === 'channel') { return channelSlug; } const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug)); return channel?.slug || null; } async function fetchTrailerUrl(videoId, entity) { const res = await unprint.get(`https://sstg.psmcode.com/?videoId=${videoId}`, { headers: { Origin: entity.origin, 'sec-fetch-site': 'cross-site', }, }); if (res.ok) { const token = res.data.token; if (token) { return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`; } } return null; } async function scrapeScene(scene, channel, parameters, includeTrailers) { const release = {}; // release.entryId = scene.id; // legacy release.entryId = scene.itemId; release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`; release.title = scene.title; release.description = scene.description; release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD'); // release.actors = scene.models?.map((model) => model.modelName) || []; release.actors = scene.models?.map((model) => ({ name: model.modelName || model.name || model.title, avatar: model.img || (parameters.avatars && `${parameters.avatars}/${slugify(model.modelName || model.name || model.title, '_')}.jpg`), url: `${channel.url}/models/${model.modelId || model.id}`, })); if (scene.img) { const poster = new URL(scene.img); release.poster = [ // scene.img.replace('med.jpg', 'hi.jpg'), // this image is not always from the same scene! for example on Petite Teens 18 scene.img, `${poster.origin}/cdn-cgi/image/width=640,quality=89${poster.pathname}`, // sometimes works when main poster is broken, observed on GotMYLF ]; } release.teaser = scene.videoTrailer; if (scene.video && includeTrailers) { // release.trailer = `https://cloudflarestream.com/${scene.video}/manifest/video.mpd?parentOrigin=${encodeURIComponent(channel.url)}`; release.trailer = await fetchTrailerUrl(scene.video, channel); } release.tags = scene.tags; release.likes = scene.stats?.likeCount; release.dislikes = scene.stats?.dislikeCount; release.channel = getChannelSlug(scene.site?.name || scene.site?.nickName, channel); return release; } function scrapeAll(scenes, channel, parameters) { return Promise.all(scenes.map(async (scene) => scrapeScene(scene, channel, parameters))); } async function fetchLatest(channel, page = 1, { parameters }) { const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); if (res.ok) { return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); } return res.status; } async function fetchLatestOrganic(channel, page, context) { const res = await http.get(`https://store.psmcdn.net/${context.parameters.endpoint}/newestMovies/items.json?orderBy="$key"&startAt="${context.cursor || 'aaaaaaaa'}"&limitToFirst=100`); if (res.ok) { const scenes = scrapeAll(Object.values(res.body), channel, context.parameters); return { // cursor implies page > 1 and first scene is last scene on previous page, // it probably won't trip up the pagination logic, but avoid the duplicate anyway scenes: context.cursor ? scenes.slice(1) : scenes, context: { cursor: Object.keys(res.body).at(-1), // official page seems to derive cursor from last scene, too }, }; } return res.status; } async function fetchLatestSearch(channel, page = 1, { parameters }) { const url = parameters.id ? `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:"${parameters.id}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}` : `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`; const res = await http.get(url); if (res.ok) { return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); } return res.status; } async function fetchScene(url, channel, baseScene, { parameters, includeTrailers }) { if (parameters.layout !== 'organic' && baseScene?.entryId && !includeTrailers) { // overview and deep data is the same in elastic API, don't hit server unnecessarily return baseScene; } const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1]; const res = await unprint.get(url, { parser: { runScripts: 'dangerously', }, }); if (res.ok) { const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent; res.context.window.fetch = () => {}; // suppress fetch missing error if (!videos) { return null; } const video = videos?.[sceneSlug] || Object.values(videos)[0]; if (video) { return scrapeScene(video, channel, parameters, includeTrailers); } return null; } return res.status; } async function scrapeProfile(actor, entity, parameters) { const profile = {}; profile.url = `${entity.url}/models/${actor.id}`; profile.description = actor.modelBio; if (actor.bio.about && !/\band\b/.test(actor.bio.about)) { const bio = actor.bio.about.split(/\n/).filter(Boolean).reduce((acc, item) => { const [key, value] = item.match(/(.+): (.+)/).slice(1); return { ...acc, [slugify(key, '_')]: value.trim() }; }, {}); // birthdate seems never/rarely correct if (bio.measurements) { profile.measurements = bio.measurements; } else { const breastSize = actor.bio.breastSize?.match(/(\d+)(\w+)/)?.slice(1) || actor.bio.about.match(/Measurements: (\d+)(\w+)/)?.slice(1); if (breastSize) { [profile.bust, profile.cup] = breastSize; } } profile.birthPlace = bio.birth_location; profile.nationality = bio.nationality; profile.ethnicity = bio.ethnicity; profile.hairColor = bio.hair_color; const piercings = actor.bio.about.match(/Piercings: (\w+)/i)?.[1]; const tattoos = actor.bio.about.match(/Tattoos: (\w+)/i)?.[1]; if (/yes|various/i.test(piercings)) profile.hasPiercings = true; else if (/no/i.test(piercings)) profile.hasPiercings = false; else if (bio.piercings) { profile.hasPiercings = true; profile.piercings = piercings; } if (/yes|various/i.test(tattoos)) profile.hasTattoos = true; else if (/no/i.test(tattoos)) profile.hasTattoos = false; else if (bio.tattoos) { profile.hasTattoos = true; profile.tattoos = tattoos; } } if (actor.bio.heightFeet && actor.bio.heightInches) { // reports 5 foot as 1 foot for some reason, but inches seem correct profile.height = convert(`${actor.bio.heightFeet >= 4 ? actor.bio.heightFeet : 5}' ${actor.bio.heightInches}"`, 'cm'); } if (actor.bio.weight) { profile.weight = convert(actor.bio.weight, 'lb', 'kg'); } profile.avatar = actor.img; profile.banner = actor.cover; profile.scenes = await Promise.all(actor.movies?.map(async (scene) => scrapeScene(scene, entity, parameters))); return profile; } async function fetchProfile(baseActor, { entity, parameters }) { // const url = format(parameters.profiles, { slug: baseActor.slug }); const url = parameters.layout === 'organic' ? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json` : `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-modelscontent`}/_doc/${parameters.modelPrefix || ''}${baseActor.slug}`; const res = await unprint.get(url); if (res.ok && res.data) { return scrapeProfile(parameters.layout === 'organic' ? res.data : res.data._source || res.body, entity, parameters); } return res.status; } module.exports = { fetchLatest, fetchScene, fetchProfile, organic: { fetchLatest: fetchLatestOrganic, fetchScene, }, search: { fetchLatest: fetchLatestSearch, fetchScene, }, };