265 lines
8.2 KiB
JavaScript
Executable File
265 lines
8.2 KiB
JavaScript
Executable File
'use strict';
|
|
|
|
const unprint = require('unprint');
|
|
|
|
const http = require('../utils/http');
|
|
const slugify = require('../utils/slugify');
|
|
const { convert } = require('../utils/convert');
|
|
|
|
function getChannelSlug(channelName, entity) {
|
|
if (!channelName) {
|
|
return null;
|
|
}
|
|
|
|
const channelSlug = slugify(channelName, '', { removePunctuation: true });
|
|
|
|
if (entity.type === 'channel') {
|
|
return channelSlug;
|
|
}
|
|
|
|
const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug));
|
|
|
|
return channel?.slug || null;
|
|
}
|
|
|
|
async function fetchTrailerUrl(videoId, entity) {
|
|
const res = await unprint.get(`https://sstg.psmcode.com/?videoId=${videoId}`, {
|
|
headers: {
|
|
Origin: entity.origin,
|
|
'sec-fetch-site': 'cross-site',
|
|
},
|
|
});
|
|
|
|
if (res.ok) {
|
|
const token = res.data.token;
|
|
|
|
if (token) {
|
|
return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async function scrapeScene(scene, channel, parameters, includeTrailers) {
|
|
const release = {};
|
|
|
|
// release.entryId = scene.id; // legacy
|
|
release.entryId = scene.itemId;
|
|
|
|
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
|
|
|
|
release.title = scene.title;
|
|
release.description = scene.description;
|
|
release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
|
|
|
|
// release.actors = scene.models?.map((model) => model.modelName) || [];
|
|
release.actors = scene.models?.map((model) => ({
|
|
name: model.modelName || model.name || model.title,
|
|
avatar: model.img || (parameters.avatars && `${parameters.avatars}/${slugify(model.modelName || model.name || model.title, '_')}.jpg`),
|
|
url: `${channel.url}/models/${model.modelId || model.id}`,
|
|
}));
|
|
|
|
if (scene.img) {
|
|
const poster = new URL(scene.img);
|
|
|
|
release.poster = [
|
|
// scene.img.replace('med.jpg', 'hi.jpg'), // this image is not always from the same scene! for example on Petite Teens 18
|
|
scene.img,
|
|
`${poster.origin}/cdn-cgi/image/width=640,quality=89${poster.pathname}`, // sometimes works when main poster is broken, observed on GotMYLF
|
|
];
|
|
}
|
|
|
|
release.teaser = scene.videoTrailer;
|
|
|
|
if (scene.video && includeTrailers) {
|
|
// release.trailer = `https://cloudflarestream.com/${scene.video}/manifest/video.mpd?parentOrigin=${encodeURIComponent(channel.url)}`;
|
|
release.trailer = await fetchTrailerUrl(scene.video, channel);
|
|
}
|
|
|
|
release.tags = scene.tags;
|
|
|
|
release.likes = scene.stats?.likeCount;
|
|
release.dislikes = scene.stats?.dislikeCount;
|
|
|
|
release.channel = getChannelSlug(scene.site?.name || scene.site?.nickName, channel);
|
|
|
|
return release;
|
|
}
|
|
|
|
function scrapeAll(scenes, channel, parameters) {
|
|
return Promise.all(scenes.map(async (scene) => scrapeScene(scene, channel, parameters)));
|
|
}
|
|
|
|
async function fetchLatest(channel, page = 1, { parameters }) {
|
|
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
|
|
|
|
if (res.ok) {
|
|
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchLatestOrganic(channel, page, context) {
|
|
const res = await http.get(`https://store.psmcdn.net/${context.parameters.endpoint}/newestMovies/items.json?orderBy="$key"&startAt="${context.cursor || 'aaaaaaaa'}"&limitToFirst=100`);
|
|
|
|
if (res.ok) {
|
|
const scenes = scrapeAll(Object.values(res.body), channel, context.parameters);
|
|
|
|
return {
|
|
// cursor implies page > 1 and first scene is last scene on previous page,
|
|
// it probably won't trip up the pagination logic, but avoid the duplicate anyway
|
|
scenes: context.cursor ? scenes.slice(1) : scenes,
|
|
context: {
|
|
cursor: Object.keys(res.body).at(-1), // official page seems to derive cursor from last scene, too
|
|
},
|
|
};
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchLatestSearch(channel, page = 1, { parameters }) {
|
|
const url = parameters.id
|
|
? `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:"${parameters.id}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
|
|
: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
|
|
|
|
const res = await http.get(url);
|
|
|
|
if (res.ok) {
|
|
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchScene(url, channel, baseScene, { parameters, includeTrailers }) {
|
|
if (parameters.layout !== 'organic' && baseScene?.entryId && !includeTrailers) {
|
|
// overview and deep data is the same in elastic API, don't hit server unnecessarily
|
|
return baseScene;
|
|
}
|
|
|
|
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
|
|
|
|
const res = await unprint.get(url, {
|
|
parser: {
|
|
runScripts: 'dangerously',
|
|
},
|
|
});
|
|
|
|
if (res.ok) {
|
|
const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent;
|
|
|
|
res.context.window.fetch = () => {}; // suppress fetch missing error
|
|
|
|
if (!videos) {
|
|
return null;
|
|
}
|
|
|
|
const video = videos?.[sceneSlug] || Object.values(videos)[0];
|
|
|
|
if (video) {
|
|
return scrapeScene(video, channel, parameters, includeTrailers);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function scrapeProfile(actor, entity, parameters) {
|
|
const profile = {};
|
|
|
|
profile.url = `${entity.url}/models/${actor.id}`;
|
|
|
|
profile.description = actor.modelBio;
|
|
|
|
if (actor.bio.about && !/\band\b/.test(actor.bio.about)) {
|
|
const bio = actor.bio.about.split(/\n/).filter(Boolean).reduce((acc, item) => {
|
|
const [key, value] = item.match(/(.+): (.+)/).slice(1);
|
|
|
|
return { ...acc, [slugify(key, '_')]: value.trim() };
|
|
}, {});
|
|
|
|
// birthdate seems never/rarely correct
|
|
|
|
if (bio.measurements) {
|
|
profile.measurements = bio.measurements;
|
|
} else {
|
|
const breastSize = actor.bio.breastSize?.match(/(\d+)(\w+)/)?.slice(1) || actor.bio.about.match(/Measurements: (\d+)(\w+)/)?.slice(1);
|
|
|
|
if (breastSize) {
|
|
[profile.bust, profile.cup] = breastSize;
|
|
}
|
|
}
|
|
|
|
profile.birthPlace = bio.birth_location;
|
|
profile.nationality = bio.nationality;
|
|
profile.ethnicity = bio.ethnicity;
|
|
profile.hairColor = bio.hair_color;
|
|
|
|
const piercings = actor.bio.about.match(/Piercings: (\w+)/i)?.[1];
|
|
const tattoos = actor.bio.about.match(/Tattoos: (\w+)/i)?.[1];
|
|
|
|
if (/yes|various/i.test(piercings)) profile.hasPiercings = true;
|
|
else if (/no/i.test(piercings)) profile.hasPiercings = false;
|
|
else if (bio.piercings) {
|
|
profile.hasPiercings = true;
|
|
profile.piercings = piercings;
|
|
}
|
|
|
|
if (/yes|various/i.test(tattoos)) profile.hasTattoos = true;
|
|
else if (/no/i.test(tattoos)) profile.hasTattoos = false;
|
|
else if (bio.tattoos) {
|
|
profile.hasTattoos = true;
|
|
profile.tattoos = tattoos;
|
|
}
|
|
}
|
|
|
|
if (actor.bio.heightFeet && actor.bio.heightInches) {
|
|
// reports 5 foot as 1 foot for some reason, but inches seem correct
|
|
profile.height = convert(`${actor.bio.heightFeet >= 4 ? actor.bio.heightFeet : 5}' ${actor.bio.heightInches}"`, 'cm');
|
|
}
|
|
|
|
if (actor.bio.weight) {
|
|
profile.weight = convert(actor.bio.weight, 'lb', 'kg');
|
|
}
|
|
|
|
profile.avatar = actor.img;
|
|
profile.banner = actor.cover;
|
|
profile.scenes = await Promise.all(actor.movies?.map(async (scene) => scrapeScene(scene, entity, parameters)));
|
|
|
|
return profile;
|
|
}
|
|
|
|
async function fetchProfile(baseActor, { entity, parameters }) {
|
|
// const url = format(parameters.profiles, { slug: baseActor.slug });
|
|
const url = parameters.layout === 'organic'
|
|
? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json`
|
|
: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-modelscontent`}/_doc/${parameters.modelPrefix || ''}${baseActor.slug}`;
|
|
|
|
const res = await unprint.get(url);
|
|
|
|
if (res.ok && res.data) {
|
|
return scrapeProfile(parameters.layout === 'organic' ? res.data : res.data._source || res.body, entity, parameters);
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
module.exports = {
|
|
fetchLatest,
|
|
fetchScene,
|
|
fetchProfile,
|
|
organic: {
|
|
fetchLatest: fetchLatestOrganic,
|
|
fetchScene,
|
|
},
|
|
search: {
|
|
fetchLatest: fetchLatestSearch,
|
|
fetchScene,
|
|
},
|
|
};
|