'use strict'; const crypto = require('crypto'); const unprint = require('unprint'); const http = require('../utils/http'); async function fetchTrailer(entryId, videoId, channel, credentials) { const url = `https://api.sysero.nl/free-stream?resource_id=${entryId}&video_id=${videoId}`; const res = await http.get(url, { headers: { Origin: channel.url, Credentials: credentials, }, }); if (res.ok) { return res.body.data?.attributes.sources.streams.mpd?.url; } return null; } // MVH's slug system seems to break on non-alphanumerical characters, but also supports ID function getSceneUrl(channel, slug, sceneId) { if (slug && /^[\w-]+$/i.test(slug)) { return `${channel.url}/sexfilms/${slug}`; } return `${channel.url}/sexfilms/${sceneId}`; } function scrapeAll(scenes, channel, context) { return scenes.reduce((acc, scene) => { const release = {}; release.entryId = scene.id; release.url = getSceneUrl(channel, scene.attributes.slug, scene.id); release.date = unprint.extractDate(scene.attributes.product.active_from, 'D/M/YY'); release.title = scene.attributes.title; release.description = scene.attributes.description; release.duration = unprint.extractDuration(scene.attributes.videos.film?.[0]?.duration); const posterPath = scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]; const teaserPath = context.clips[scene.relationships.clips?.data[0]?.id]; if (posterPath) { release.poster = `https://cdndo.sysero.nl${scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]}`; } if (scene.attributes.videos.trailer?.[0]) { release.trailer = async () => fetchTrailer(scene.id, scene.attributes.videos.trailer[0].id, channel, context.credentials); } if (teaserPath) { release.teaser = `https://cdndo.sysero.nl${teaserPath}`; } release.tags = scene.relationships.categories?.data.map((category) => context.tags[category.id]?.replace(/-/g, ' ')).filter(Boolean); release.language = scene.attributes.videos.film?.[0]?.language; if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) { // all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen return { ...acc, unextracted: [...acc.unextracted, release] }; } return { ...acc, scenes: [...acc.scenes, release] }; }, { scenes: [], unextracted: [], }); } function getCredentials(channel) { const now = Math.floor(Date.now() / 1000); const hash = crypto .createHmac('sha256', channel.parameters.secret) .update(`${channel.parameters.frontend}${now.toString()}`) .digest('hex'); const credentials = `Syserauth ${channel.parameters.frontend}-${hash}-${now.toString(16)}`; return credentials; } const falseCountry = /afghanistan/i; // no country defaults to Afghanistan function getLocation(model) { const country = model.country && !falseCountry.test(model.country) ? model.country : null; return [model.city, model.county, country] .map((segment) => segment?.trim()) .filter(Boolean) .join(', ') || null; } function scrapeProfile(model, { entity, includeScenes = true }) { const actor = {}; actor.name = model.title; actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url); actor.entryId = model.id; actor.description = model.description; actor.dateOfBirth = model.birth_date && model.age > 18 ? new Date(model.birth_date) : null; // sometimes seems to be profile creation date actor.age = model.age > 18 ? model.age : null; actor.orientation = model.sexual_orientation; actor.birthPlace = getLocation(model); actor.height = Number(model.length) || null; actor.weight = Number(model.weight) || null; actor.eyes = model.eye_color; actor.hairColor = model.hair_color; if (includeScenes) { actor.scenes = model.videos?.map((video) => ({ entryId: video.id, url: getSceneUrl(entity, video.slug, video.id), title: video.title, description: video.description, })); } actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl'); return actor; } function scrapeSceneData(scene, { entity }) { const release = {}; release.entryId = scene.id; release.url = getSceneUrl(entity, scene.slug, scene.id); release.title = scene.title; release.description = scene.description; release.date = scene.uploadDate ? new Date(scene.uploadDate) : unprint.extractDate(scene.product.active_from, 'D/M/YY'); release.actors = scene.models?.map((model) => scrapeProfile(model, { entity, includeScenes: false })); release.duration = scene.seconds || unprint.extractTimestamp(scene.isoDuration) || Number(scene.video_paid?.duration) * 60; release.tags = scene.categories?.map((category) => category.slug.replace(/-/g, ' ')); if (scene.thumb) { release.poster = [ scene.thumb.original, scene.thumb.xxl, scene.thumb.xl, // ... l, m, s, xs, xxs, probably little point trying all of them ].map((poster) => unprint.prefixUrl(poster, 'https://cdndo.sysero.nl')); } release.photos = scene.gallery; if (scene.trailer) { release.trailer = async () => { const credentials = getCredentials(entity); return fetchTrailer(scene.id, scene.trailer.id, entity, credentials); }; } return release; } function scrapeScene({ _query, window }, context) { const data = window.__NUXT__?.state?.videoStore?.video; if (data) { return scrapeSceneData(data, context); } return null; } async function fetchLatest(channel, page, context) { const credentials = getCredentials(channel); const res = await http.get(`https://api.sysero.nl/videos?page=${page}&count=20&type=video&include=images:types(thumb|thumb_mobile),categories,clips&filter[status]=published&filter[products]=1%2C2&sort[published_at]=DESC&frontend=${channel.parameters.frontend}`, { headers: { Origin: channel.url, Credentials: credentials, }, }); if (res.ok && res.body.data) { const tags = Object.fromEntries(res.body.included?.filter((item) => item.type === 'category').map((item) => [item.id, item.attributes.slug]) || []); const images = Object.fromEntries(res.body.included?.filter((item) => item.type === 'image' && item.attributes.types === 'thumb').map((item) => [item.id, item.attributes.path]) || []); const clips = Object.fromEntries(res.body.included?.filter((item) => item.type === 'clip').map((item) => [item.id, item.attributes.path]) || []); return scrapeAll(res.body.data, channel, { ...context, images, clips, tags, credentials }); } return res.status; } async function fetchProfile(actor, { entity }) { const credentials = getCredentials(entity); const url = `${entity.url}/modellen/${actor.slug}`; const res = await unprint.get(url, { headers: { Origin: entity.url, Credentials: credentials, }, parser: { runScripts: 'dangerously', }, }); if (res.ok) { const data = res.context.window.__NUXT__?.state?.modelStore?.model; if (data) { return scrapeProfile(data, { entity }); } return null; } return res.status; } module.exports = { fetchLatest, fetchProfile, scrapeScene: { scraper: scrapeScene, unprint: true, parser: { runScripts: 'dangerously', }, }, };