243 lines
7.0 KiB
JavaScript
243 lines
7.0 KiB
JavaScript
'use strict';
|
|
|
|
const crypto = require('crypto');
|
|
const unprint = require('unprint');
|
|
|
|
const http = require('../utils/http');
|
|
|
|
async function fetchTrailer(entryId, videoId, channel, credentials) {
|
|
const url = `https://api.sysero.nl/free-stream?resource_id=${entryId}&video_id=${videoId}`;
|
|
|
|
const res = await http.get(url, {
|
|
headers: {
|
|
Origin: channel.url,
|
|
Credentials: credentials,
|
|
},
|
|
});
|
|
|
|
if (res.ok) {
|
|
return res.body.data?.attributes.sources.streams.mpd?.url;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
// MVH's slug system seems to break on non-alphanumerical characters, but also supports ID
|
|
function getSceneUrl(channel, slug, sceneId) {
|
|
if (slug && /^[\w-]+$/i.test(slug)) {
|
|
return `${channel.url}/sexfilms/${slug}`;
|
|
}
|
|
|
|
return `${channel.url}/sexfilms/${sceneId}`;
|
|
}
|
|
|
|
function scrapeAll(scenes, channel, context) {
|
|
return scenes.reduce((acc, scene) => {
|
|
const release = {};
|
|
|
|
release.entryId = scene.id;
|
|
release.url = getSceneUrl(channel, scene.attributes.slug, scene.id);
|
|
release.date = unprint.extractDate(scene.attributes.product.active_from, 'D/M/YY');
|
|
|
|
release.title = scene.attributes.title;
|
|
release.description = scene.attributes.description;
|
|
release.duration = unprint.extractDuration(scene.attributes.videos.film?.[0]?.duration);
|
|
|
|
const posterPath = scene.attributes.images.thumb?.[0]?.path || context.images[scene.id];
|
|
const teaserPath = context.clips[scene.relationships.clips?.data[0]?.id];
|
|
|
|
if (posterPath) {
|
|
release.poster = `https://cdndo.sysero.nl${scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]}`;
|
|
}
|
|
|
|
if (scene.attributes.videos.trailer?.[0]) {
|
|
release.trailer = async () => fetchTrailer(scene.id, scene.attributes.videos.trailer[0].id, channel, context.credentials);
|
|
}
|
|
|
|
if (teaserPath) {
|
|
release.teaser = `https://cdndo.sysero.nl${teaserPath}`;
|
|
}
|
|
|
|
release.tags = scene.relationships.categories?.data.map((category) => context.tags[category.id]?.replace(/-/g, ' ')).filter(Boolean);
|
|
release.language = scene.attributes.videos.film?.[0]?.language;
|
|
|
|
if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) {
|
|
// all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen
|
|
return { ...acc, unextracted: [...acc.unextracted, release] };
|
|
}
|
|
|
|
return { ...acc, scenes: [...acc.scenes, release] };
|
|
}, {
|
|
scenes: [],
|
|
unextracted: [],
|
|
});
|
|
}
|
|
|
|
function getCredentials(channel) {
|
|
const now = Math.floor(Date.now() / 1000);
|
|
|
|
const hash = crypto
|
|
.createHmac('sha256', channel.parameters.secret)
|
|
.update(`${channel.parameters.frontend}${now.toString()}`)
|
|
.digest('hex');
|
|
|
|
const credentials = `Syserauth ${channel.parameters.frontend}-${hash}-${now.toString(16)}`;
|
|
|
|
return credentials;
|
|
}
|
|
|
|
const falseCountry = /afghanistan/i; // no country defaults to Afghanistan
|
|
|
|
function getLocation(model) {
|
|
const country = model.country && !falseCountry.test(model.country) ? model.country : null;
|
|
|
|
return [model.city, model.county, country]
|
|
.map((segment) => segment?.trim())
|
|
.filter(Boolean)
|
|
.join(', ') || null;
|
|
}
|
|
|
|
function scrapeProfile(model, { entity, includeScenes = true }) {
|
|
const actor = {};
|
|
|
|
actor.name = model.title;
|
|
actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url);
|
|
|
|
actor.entryId = model.id;
|
|
|
|
actor.description = model.description;
|
|
|
|
actor.dateOfBirth = model.birth_date && model.age > 18 ? new Date(model.birth_date) : null; // sometimes seems to be profile creation date
|
|
actor.age = model.age > 18 ? model.age : null;
|
|
actor.orientation = model.sexual_orientation;
|
|
|
|
actor.birthPlace = getLocation(model);
|
|
|
|
actor.height = Number(model.length) || null;
|
|
actor.weight = Number(model.weight) || null;
|
|
|
|
actor.eyes = model.eye_color;
|
|
actor.hairColor = model.hair_color;
|
|
|
|
if (includeScenes) {
|
|
actor.scenes = model.videos?.map((video) => ({
|
|
entryId: video.id,
|
|
url: getSceneUrl(entity, video.slug, video.id),
|
|
title: video.title,
|
|
description: video.description,
|
|
}));
|
|
}
|
|
|
|
actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl');
|
|
|
|
return actor;
|
|
}
|
|
|
|
function scrapeSceneData(scene, { entity }) {
|
|
const release = {};
|
|
|
|
release.entryId = scene.id;
|
|
release.url = getSceneUrl(entity, scene.slug, scene.id);
|
|
|
|
release.title = scene.title;
|
|
release.description = scene.description;
|
|
release.date = scene.uploadDate
|
|
? new Date(scene.uploadDate)
|
|
: unprint.extractDate(scene.product.active_from, 'D/M/YY');
|
|
|
|
release.actors = scene.models?.map((model) => scrapeProfile(model, { entity, includeScenes: false }));
|
|
|
|
release.duration = scene.seconds || unprint.extractTimestamp(scene.isoDuration) || Number(scene.video_paid?.duration) * 60;
|
|
release.tags = scene.categories?.map((category) => category.slug.replace(/-/g, ' '));
|
|
|
|
if (scene.thumb) {
|
|
release.poster = [
|
|
scene.thumb.original,
|
|
scene.thumb.xxl,
|
|
scene.thumb.xl,
|
|
// ... l, m, s, xs, xxs, probably little point trying all of them
|
|
].map((poster) => unprint.prefixUrl(poster, 'https://cdndo.sysero.nl'));
|
|
}
|
|
|
|
release.photos = scene.gallery;
|
|
|
|
if (scene.trailer) {
|
|
release.trailer = async () => {
|
|
const credentials = getCredentials(entity);
|
|
return fetchTrailer(scene.id, scene.trailer.id, entity, credentials);
|
|
};
|
|
}
|
|
|
|
return release;
|
|
}
|
|
|
|
function scrapeScene({ _query, window }, context) {
|
|
const data = window.__NUXT__?.state?.videoStore?.video;
|
|
|
|
if (data) {
|
|
return scrapeSceneData(data, context);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async function fetchLatest(channel, page, context) {
|
|
const credentials = getCredentials(channel);
|
|
|
|
const res = await http.get(`https://api.sysero.nl/videos?page=${page}&count=20&type=video&include=images:types(thumb|thumb_mobile),categories,clips&filter[status]=published&filter[products]=1%2C2&sort[published_at]=DESC&frontend=${channel.parameters.frontend}`, {
|
|
headers: {
|
|
Origin: channel.url,
|
|
Credentials: credentials,
|
|
},
|
|
});
|
|
|
|
if (res.ok && res.body.data) {
|
|
const tags = Object.fromEntries(res.body.included?.filter((item) => item.type === 'category').map((item) => [item.id, item.attributes.slug]) || []);
|
|
const images = Object.fromEntries(res.body.included?.filter((item) => item.type === 'image' && item.attributes.types === 'thumb').map((item) => [item.id, item.attributes.path]) || []);
|
|
const clips = Object.fromEntries(res.body.included?.filter((item) => item.type === 'clip').map((item) => [item.id, item.attributes.path]) || []);
|
|
|
|
return scrapeAll(res.body.data, channel, { ...context, images, clips, tags, credentials });
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
async function fetchProfile(actor, { entity }) {
|
|
const credentials = getCredentials(entity);
|
|
const url = `${entity.url}/modellen/${actor.slug}`;
|
|
|
|
const res = await unprint.get(url, {
|
|
headers: {
|
|
Origin: entity.url,
|
|
Credentials: credentials,
|
|
},
|
|
parser: {
|
|
runScripts: 'dangerously',
|
|
},
|
|
});
|
|
|
|
if (res.ok) {
|
|
const data = res.context.window.__NUXT__?.state?.modelStore?.model;
|
|
|
|
if (data) {
|
|
return scrapeProfile(data, { entity });
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
return res.status;
|
|
}
|
|
|
|
module.exports = {
|
|
fetchLatest,
|
|
fetchProfile,
|
|
scrapeScene: {
|
|
scraper: scrapeScene,
|
|
unprint: true,
|
|
parser: {
|
|
runScripts: 'dangerously',
|
|
},
|
|
},
|
|
};
|