DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated Team Skeet and MYLF.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2023-08-11 03:56:08 +02:00
parent 71efd7a96f
commit 02850eb6e3
612 changed files with 3074 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/scrapers/scrapers.js
View File

@@ -148,6 +148,7 @@ const scrapers = {
purgatoryx,
radical,
rickysroom,
sayuncle: teamskeet,
score,
sexyhub: mindgeek,
spizoo,
@@ -285,6 +286,7 @@ const scrapers = {
realvr: badoink,
rickysroom,
roccosiffredi: famedigital,
sayuncle: teamskeet,
score,
seehimfuck: hush,
sexyhub: mindgeek,

86
src/scrapers/teamskeet.js
View File

@@ -1,7 +1,5 @@
'use strict';
const format = require('template-format');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
@@ -12,26 +10,31 @@ function getChannelSlug(channelName, entity) {
return entity.slug;
}
if (!channelName) {
return null;
}
const channelSlug = slugify(channelName, '', { removePunctuation: true });
const channel = entity.children.find((child) => new RegExp(channelSlug).test(child.slug));
return channel?.slug || null;
}
function scrapeScene(scene, channel) {
function scrapeScene(scene, channel, parameters) {
const release = {};
release.entryId = scene.id;
release.url = `${channel.type === 'network' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
release.title = scene.title;
release.description = scene.description;
release.date = qu.extractDate(scene.publishedDate);
release.actors = scene.models?.map((model) => model.modelName) || [];
release.actors = scene.models?.map((model) => ({
name: model.modelName,
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
avatar: parameters.avatars && `${parameters.avatars}/${slugify(model.modelName, '_')}.jpg`,
url: `${channel.url}/models/${model.modelId}`,
}));
release.poster = [
@@ -42,26 +45,30 @@ function scrapeScene(scene, channel) {
release.teaser = scene.videoTrailer;
if (scene.video) {
release.trailer = { stream: `https://videodelivery.net/${scene.video}/manifest/video.mpd` };
release.trailer = `https://cloudflarestream.com/${scene.video}/manifest/video.mpd?parentOrigin=${encodeURIComponent(channel.url)}`;
}
release.tags = scene.tags;
release.likes = scene.stats.likeCount;
release.dislikes = scene.stats.dislikeCount;
release.likes = scene.stats?.likeCount;
release.dislikes = scene.stats?.dislikeCount;
release.channel = getChannelSlug(scene.site.name || scene.site.nickName, channel);
release.channel = getChannelSlug(scene.site?.name || scene.site?.nickName, channel);
return release;
}
function scrapeAll(scenes, channel) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, channel));
function scrapeAll(scenes, channel, parameters) {
return scenes.map((scene) => scrapeScene(scene, channel, parameters));
}
function scrapeProfile(actor, entity) {
function scrapeProfile(actor, entity, parameters) {
const profile = {};
profile.url = `${entity.url}/models/${actor.id}`;
profile.description = actor.modelBio;
if (actor.bio.about && !/\band\b/.test(actor.bio.about)) {
const bio = actor.bio.about.split(/\n/).filter(Boolean).reduce((acc, item) => {
const [key, value] = item.match(/(.+): (.+)/).slice(1);
@@ -113,43 +120,74 @@ function scrapeProfile(actor, entity) {
}
profile.avatar = actor.img;
profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity));
profile.banner = actor.cover;
profile.scenes = actor.movies?.map((scene) => scrapeScene(scene, entity, parameters));
return profile;
}
async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
const res = await http.get(`https://store2.psmcdn.net/${parameters.endpoint}-videoscontent/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
if (res.ok) {
return scrapeAll(res.body.hits.hits, channel);
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
}
return res.status;
}
async function fetchLatestOrganic(channel, page, context) {
const res = await http.get(`https://store.psmcdn.net/${context.parameters.endpoint}/newestMovies/items.json?orderBy="$key"&startAt="${context.cursor || 'aaaaaaaa'}"&limitToFirst=100`);
if (res.ok) {
const scenes = scrapeAll(Object.values(res.body), channel, context.parameters);
return {
// cursor implies page > 1 and first scene is last scene on previous page,
// it probably won't trip up the pagination logic, but avoid the duplicate anyway
scenes: context.cursor ? scenes.slice(1) : scenes,
context: {
cursor: Object.keys(res.body).at(-1), // official page seems to derive cursor from last scene, too
},
};
}
return res.status;
}
async function fetchScene(url, channel, baseScene, { parameters }) {
if (baseScene?.entryId) {
// overview and deep data is the same, don't hit server unnecessarily
if (parameters.layout !== 'organic' && baseScene?.entryId) {
// overview and deep data is the same in elastic API, don't hit server unnecessarily
return baseScene;
}
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
const res = await http.get(`${parameters.videos}/${sceneSlug}`);
const res = await http.get(parameters.layout === 'organic'
? `https://store.psmcdn.net/${parameters.endpoint}/moviesContent/${sceneSlug}.json`
: `https://store2.psmcdn.net/${parameters.endpoint}-videoscontent/_doc/${sceneSlug}`);
if (res.ok && res.body.found) {
return scrapeScene(res.body._source, channel);
return scrapeScene(res.body._source, channel, parameters);
}
if (res.ok && parameters.layout === 'organic' && res.body.id) {
return scrapeScene(res.body, channel, parameters);
}
return res.status;
}
async function fetchProfile(baseActor, { entity, parameters }) {
const url = format(parameters.profiles, { slug: baseActor.slug });
// const url = format(parameters.profiles, { slug: baseActor.slug });
const url = parameters.layout === 'organic'
? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json`
: `https://store2.psmcdn.net/${parameters.endpoint}-modelscontent/_doc/${baseActor.slug}`;
const res = await qu.get(url);
if (res.ok && res.body) {
return scrapeProfile(res.body._source || res.body, entity);
return scrapeProfile(parameters.layout === 'organic' ? res.body : res.body._source || res.body, entity, parameters);
}
return res.status;
@@ -159,4 +197,8 @@ module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
organic: {
fetchLatest: fetchLatestOrganic,
fetchScene,
},
};