DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Generalized Team Skeet scraper, added MYLF network and various Team Skeet partner channels.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-03-29 22:22:56 +02:00
parent d17dbf1b36
commit c386a9098f
298 changed files with 456 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/scrapers/teamskeet.js
View File

@@ -1,70 +1,65 @@
'use strict';
const format = require('template-format');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
function scrapeAll(scenes) {
return scenes.map((scene) => {
const release = {};
function getChannelSlug(channelName, entity) {
if (entity.type === 'channel') {
return entity.slug;
}
release.entryId = scene.id;
release.url = `https://teamskeet.com/movies/${release.entryId}`;
const channelSlug = slugify(channelName, '', { removePunctuation: true });
const channel = entity.children.find(child => new RegExp(channelSlug).test(child.slug));
release.title = scene.title;
release.date = qu.extractDate(scene.publishedDate);
release.actors = scene.models?.map(model => model.modelName) || [];
release.poster = [
scene.img.replace('med.jpg', 'hi.jpg'),
scene.img,
];
release.teaser = scene.videoTrailer;
if (scene.video) {
release.trailer = { stream: `https://videodelivery.net/${scene.video}/manifest/video.mpd` };
}
release.likes = scene.stats.likeCount;
release.dislikes = scene.stats.dislikeCount;
release.channel = slugify(scene.site.name, '')
.replace('hobybuchanon', 'tshobybuchanon'); // slug collision with his own site
return release;
});
return channel.slug;
}
function scrapeScene(scene) {
function scrapeScene(scene, channel) {
const release = {};
release.entryId = scene.id;
release.url = `${channel.type === 'network' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
release.title = scene.title;
release.description = scene.description;
release.date = qu.extractDate(scene.publishedDate);
release.actors = scene.models?.map(model => model.modelName) || [];
release.actors = scene.models?.map(model => ({
name: model.modelName,
avatar: `https://images.mylfcdn.net/tsv4/model/profiles/${slugify(model.modelName, '_')}.jpg`,
url: `${channel.url}/models/www.mylf.com/models/${model.modelId}`,
}));
release.poster = [
scene.img.replace('med.jpg', 'hi.jpg'),
scene.img,
];
release.channel = slugify(scene.site.name, '')
.replace('hobybuchanon', 'tshobybuchanon'); // slug collision with his own site
release.teaser = scene.videoTrailer;
if (scene.video) {
release.trailer = { stream: `https://videodelivery.net/${scene.video}/manifest/video.mpd` };
}
release.tags = scene.tags;
release.likes = scene.stats.likeCount;
release.dislikes = scene.stats.dislikeCount;
release.channel = getChannelSlug(scene.site.name || scene.site.nickName, channel);
return release;
}
function scrapeProfile(actor) {
function scrapeAll(scenes, channel) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, channel));
}
function scrapeProfile(actor, entity) {
const profile = {};
if (actor.bio.about && !/\band\b/.test(actor.bio.about)) {
@@ -119,45 +114,38 @@ function scrapeProfile(actor) {
}
profile.avatar = actor.img;
profile.releases = scrapeAll(actor.movies);
profile.scenes = actor.movies?.map(scene => scrapeScene(scene, entity));
return profile;
}
async function fetchLatest(channel, _page = 1) {
// freshman year, layna landry
if (!channel.parameters?.id) {
return null;
}
const url = `https://store.psmcdn.net/ts-organic-iiiokv9kyo/seriesContent/${channel.parameters.id}/latestMovies.json`;
const res = await http.get(url);
async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
if (res.ok) {
return scrapeAll(Object.values(res.body), channel);
return scrapeAll(res.body.hits.hits, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const entryId = new URL(url).pathname.match(/\/movies\/(.+)$/)[1];
const apiUrl = `https://store.psmcdn.net/ts-organic-iiiokv9kyo/videosContent/${entryId}.json`;
async function fetchScene(url, channel, baseScene, { parameters }) {
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
const res = await http.get(`${parameters.videos}/${sceneSlug}`);
const res = await http.get(apiUrl);
if (res.ok) {
return scrapeScene(res.body, channel);
if (res.ok && res.body.found) {
return scrapeScene(res.body._source, channel);
}
return res.status;
}
async function fetchProfile(baseActor) {
const res = await http.get(`https://store.psmcdn.net/ts-organic-iiiokv9kyo/modelsContent/${slugify(baseActor.name)}.json`);
async function fetchProfile(baseActor, { entity, parameters }) {
const url = format(parameters.profiles, { slug: baseActor.slug });
const res = await qu.get(url);
if (res.ok && res.body) {
return scrapeProfile(res.body);
if (res.ok) {
return scrapeProfile(res.body, entity);
}
return res.status;