Merged migrations.
This commit is contained in:
DebaucheryLibrarian
@@ -1,106 +1,124 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const http = require('../utils/http');
|
||||
async function fetchTrailer(entryId, videoId, channel, { parameters }) {
|
||||
// query seems superfluous, but mimics native behavior
|
||||
const url = `https://apiv2.sysero.nl/api/mvh/stream/get/${entryId}/${videoId}?query=(include:(source:()))`;
|
||||
|
||||
async function fetchTrailer(entryId, videoId, channel, credentials) {
|
||||
const url = `https://api.sysero.nl/free-stream?resource_id=${entryId}&video_id=${videoId}`;
|
||||
|
||||
const res = await http.get(url, {
|
||||
const res = await unprint.get(url, {
|
||||
headers: {
|
||||
Origin: channel.url,
|
||||
Credentials: credentials,
|
||||
Origin: channel.origin,
|
||||
Credentials: `sysero ${parameters.credentials}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return res.body.data?.attributes.sources.streams.mpd?.url;
|
||||
return res.data.data?.ITEM?.LINKS?.map((link) => ({
|
||||
stream: link.URL,
|
||||
quality: Number(link.HEIGHT) || null,
|
||||
}));
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// MVH's slug system seems to break on non-alphanumerical characters, but also supports ID
|
||||
function getSceneUrl(channel, slug, sceneId) {
|
||||
if (slug && /^[\w-]+$/i.test(slug)) {
|
||||
return `${channel.url}/sexfilms/${slug}`;
|
||||
async function scrapeSceneData(scene, channel, context, isDeep) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = scene.id;
|
||||
release.url = `${channel.origin}/sexfilms/${scene.slug}`;
|
||||
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
|
||||
release.date = unprint.extractDate(scene.active_from, 'YYYY-MM-DD HH:mm:ss');
|
||||
release.duration = scene.videos?.data?.film?.[0]?.duration;
|
||||
|
||||
release.actors = scene.model?.map((model) => ({
|
||||
entryId: model.id,
|
||||
name: model.title,
|
||||
url: model.slug && `${channel.origin}/modellen/${model.slug}`,
|
||||
}));
|
||||
|
||||
release.tags = scene.resources?.data?.filter((tag) => tag.type === 'category').map((tag) => tag.title);
|
||||
release.language = scene.videos?.data?.film?.[0]?.language;
|
||||
|
||||
if (isDeep) {
|
||||
const thumb = scene.images?.data?.thumb?.[0];
|
||||
|
||||
if (thumb) {
|
||||
release.poster = `https://cdndo.sysero.nl${thumb.path}`;
|
||||
}
|
||||
|
||||
release.photos = scene.images?.data?.album?.map((image) => `https://cdndo.sysero.nl${image.path}`) || [];
|
||||
|
||||
const videoId = scene.videos?.data?.trailer?.[0]?.id;
|
||||
|
||||
if (context.include.trailers && videoId) {
|
||||
release.trailer = await fetchTrailer(release.entryId, videoId, channel, context);
|
||||
}
|
||||
} else {
|
||||
[release.poster, ...release.photos] = scene.images?.data?.thumb?.map((image) => `https://cdndo.sysero.nl${image.path}`) || [];
|
||||
}
|
||||
|
||||
return `${channel.url}/sexfilms/${sceneId}`;
|
||||
if (scene.clips?.data?.[0]) {
|
||||
release.teaser = `https://cdndo.sysero.nl${scene.clips.data[0].path}`;
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, channel, context) {
|
||||
return scenes.reduce((acc, scene) => {
|
||||
const release = {};
|
||||
return scenes.reduce(async (chain, scene) => {
|
||||
const acc = await chain;
|
||||
|
||||
/*
|
||||
release.entryId = scene.id;
|
||||
release.url = getSceneUrl(channel, scene.attributes.slug, scene.id);
|
||||
release.date = unprint.extractDate(scene.attributes.product.active_from, 'D/M/YY');
|
||||
|
||||
release.title = scene.attributes.title;
|
||||
release.description = scene.attributes.description;
|
||||
release.duration = unprint.extractDuration(scene.attributes.videos.film?.[0]?.duration);
|
||||
|
||||
const posterPath = scene.attributes.images.thumb?.[0]?.path || context.images[scene.id];
|
||||
const teaserPath = context.clips[scene.relationships.clips?.data[0]?.id];
|
||||
|
||||
if (posterPath) {
|
||||
release.poster = `https://cdndo.sysero.nl${scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]}`;
|
||||
if (!scene.slug) {
|
||||
// page 2 of Vurig Vlaanderen and possible others return broken entries, even on website
|
||||
return acc;
|
||||
}
|
||||
|
||||
if (teaserPath) {
|
||||
release.teaser = `https://cdndo.sysero.nl${teaserPath}`;
|
||||
}
|
||||
|
||||
release.tags = scene.relationships.categories?.data.map((category) => context.tags[category.id]?.replace(/-/g, ' ')).filter(Boolean);
|
||||
release.language = scene.attributes.videos.film?.[0]?.language;
|
||||
const release = await scrapeSceneData(scene, channel, context, false);
|
||||
|
||||
if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) {
|
||||
// all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen
|
||||
// the international releases should go on MVH, but the API can't filter for NL+EN, so we do it here
|
||||
return { ...acc, unextracted: [...acc.unextracted, release] };
|
||||
}
|
||||
*/
|
||||
|
||||
console.log(scene);
|
||||
console.log(release);
|
||||
|
||||
return { ...acc, scenes: [...acc.scenes, release] };
|
||||
}, {
|
||||
}, Promise.resolve({
|
||||
scenes: [],
|
||||
unextracted: [],
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page, context) {
|
||||
// query seems to break if any component is left out or even moved
|
||||
const res = await http.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
|
||||
// const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
|
||||
// const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
|
||||
const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
|
||||
headers: {
|
||||
Origin: channel.origin,
|
||||
Credentials: context.parameters.credentials,
|
||||
Credentials: `sysero ${context.parameters.credentials}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.body.data) {
|
||||
return scrapeAll(res.body.data, channel, context);
|
||||
if (res.ok && res.data.data) {
|
||||
return scrapeAll(res.data.data, channel, context);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function getCredentials(channel) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
async function scrapeScene({ window }, context) {
|
||||
const data = window.__NUXT__?.state?.resourcesStore?.video;
|
||||
|
||||
const hash = crypto
|
||||
.createHmac('sha256', channel.parameters.secret)
|
||||
.update(`${channel.parameters.frontend}${now.toString()}`)
|
||||
.digest('hex');
|
||||
if (data) {
|
||||
return scrapeSceneData(data, context.entity, context, true);
|
||||
}
|
||||
|
||||
const credentials = `Syserauth ${channel.parameters.frontend}-${hash}-${now.toString(16)}`;
|
||||
|
||||
return credentials;
|
||||
return null;
|
||||
}
|
||||
|
||||
const falseCountry = /afghanistan/i; // no country defaults to Afghanistan
|
||||
@@ -114,9 +132,11 @@ function getLocation(model) {
|
||||
.join(', ') || null;
|
||||
}
|
||||
|
||||
function scrapeProfile(model, { entity, includeScenes = true }) {
|
||||
async function scrapeProfile(model, { entity }) {
|
||||
const actor = {};
|
||||
|
||||
// gender unreliable, seems to report everyone as 'vrouw' (woman)
|
||||
|
||||
actor.name = model.title;
|
||||
actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url);
|
||||
|
||||
@@ -136,76 +156,20 @@ function scrapeProfile(model, { entity, includeScenes = true }) {
|
||||
actor.eyes = model.eye_color;
|
||||
actor.hairColor = model.hair_color;
|
||||
|
||||
if (includeScenes) {
|
||||
actor.scenes = model.videos?.map((video) => ({
|
||||
entryId: video.id,
|
||||
url: getSceneUrl(entity, video.slug, video.id),
|
||||
title: video.title,
|
||||
description: video.description,
|
||||
}));
|
||||
}
|
||||
|
||||
actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl');
|
||||
|
||||
actor.scenes = await Promise.all(model.video?.map(async (video) => scrapeSceneData(video, entity, false)));
|
||||
|
||||
return actor;
|
||||
}
|
||||
|
||||
function scrapeSceneData(scene, { entity }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = scene.id;
|
||||
release.url = getSceneUrl(entity, scene.slug, scene.id);
|
||||
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
release.date = scene.uploadDate
|
||||
? new Date(scene.uploadDate)
|
||||
: unprint.extractDate(scene.product.active_from, 'D/M/YY');
|
||||
|
||||
release.actors = scene.models?.map((model) => scrapeProfile(model, { entity, includeScenes: false }));
|
||||
|
||||
release.duration = scene.seconds || unprint.extractTimestamp(scene.isoDuration) || Number(scene.video_paid?.duration) * 60;
|
||||
release.tags = scene.categories?.map((category) => category.slug.replace(/-/g, ' '));
|
||||
|
||||
if (scene.thumb) {
|
||||
release.poster = [
|
||||
scene.thumb.original,
|
||||
scene.thumb.xxl,
|
||||
scene.thumb.xl,
|
||||
// ... l, m, s, xs, xxs, probably little point trying all of them
|
||||
].map((poster) => unprint.prefixUrl(poster, 'https://cdndo.sysero.nl'));
|
||||
}
|
||||
|
||||
release.photos = scene.gallery;
|
||||
|
||||
if (scene.trailer) {
|
||||
release.trailer = async () => {
|
||||
const credentials = getCredentials(entity);
|
||||
return fetchTrailer(scene.id, scene.trailer.id, entity, credentials);
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeScene({ _query, window }, context) {
|
||||
const data = window.__NUXT__?.state?.videoStore?.video;
|
||||
|
||||
if (data) {
|
||||
return scrapeSceneData(data, context);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchProfile(actor, { entity }) {
|
||||
const credentials = getCredentials(entity);
|
||||
const url = `${entity.url}/modellen/${actor.slug}`;
|
||||
async function fetchProfile(actor, { entity, parameters }) {
|
||||
const url = actor.url || `${entity.url}/modellen/${actor.slug}`;
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
headers: {
|
||||
Origin: entity.url,
|
||||
Credentials: credentials,
|
||||
Origin: entity.origin,
|
||||
Credentials: `sysero ${parameters.credentials}`,
|
||||
},
|
||||
parser: {
|
||||
runScripts: 'dangerously',
|
||||
|
||||
@@ -76,7 +76,7 @@ function withRelations(queryBuilder, withMedia) {
|
||||
async function matchReleaseTags(releases) {
|
||||
const tags = releases
|
||||
.map((release) => release.tags).flat()
|
||||
.map((tag) => tag?.trim().toLowerCase())
|
||||
.map((tag) => tag?.trim().match(/[a-z0-9]+/ig)?.join(' ').toLowerCase())
|
||||
.filter(Boolean);
|
||||
|
||||
const tagEntries = await knex('tags')
|
||||
|
||||
Reference in New Issue
Block a user