DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Cleaned up Team Skeet config, added missing Say Uncle channels.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-02-06 01:48:07 +01:00
parent a05928e399
commit b01913690e
4 changed files with 665 additions and 1584 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/scrapers/teamskeet.js
View File

@@ -41,7 +41,7 @@ async function fetchTrailerUrl(videoId, entity) {
return null;
}
async function scrapeScene(scene, channel, parameters, includeTrailers) {
async function scrapeScene(scene, channel, _parameters, includeTrailers) {
const release = {};
// release.entryId = scene.id; // legacy
@@ -53,10 +53,10 @@ async function scrapeScene(scene, channel, parameters, includeTrailers) {
release.description = scene.description;
release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
// release.actors = scene.models?.map((model) => model.modelName) || [];
release.actors = scene.models?.map((model) => ({
name: model.modelName || model.name || model.title,
avatar: model.img || (parameters.avatars && `${parameters.avatars}/${slugify(model.modelName || model.name || model.title, '_')}.jpg`),
avatar: model.img,
gender: model.gender,
url: `${channel.url}/models/${model.modelId || model.id}`,
}));
@@ -92,38 +92,12 @@ function scrapeAll(scenes, channel, parameters) {
}
async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
// url: 'https://www.pervz.com/series/pervz-features',
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
}
return res.status;
}
async function fetchLatestOrganic(channel, page, context) {
const res = await http.get(`https://store.psmcdn.net/${context.parameters.endpoint}/newestMovies/items.json?orderBy="$key"&startAt="${context.cursor || 'aaaaaaaa'}"&limitToFirst=100`);
if (res.ok) {
const scenes = scrapeAll(Object.values(res.body), channel, context.parameters);
return {
// cursor implies page > 1 and first scene is last scene on previous page,
// it probably won't trip up the pagination logic, but avoid the duplicate anyway
scenes: context.cursor ? scenes.slice(1) : scenes,
context: {
cursor: Object.keys(res.body).at(-1), // official page seems to derive cursor from last scene, too
},
};
}
return res.status;
}
async function fetchLatestSearch(channel, page = 1, { parameters }) {
const url = parameters.id
? `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:"${parameters.id}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
const url = seriesId
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
const res = await http.get(url);
@@ -135,7 +109,7 @@ async function fetchLatestSearch(channel, page = 1, { parameters }) {
}
async function fetchScene(url, channel, baseScene, { parameters, includeTrailers }) {
if (parameters.layout !== 'organic' && baseScene?.entryId && !includeTrailers) {
if (baseScene?.entryId && !includeTrailers) {
// overview and deep data is the same in elastic API, don't hit server unnecessarily
return baseScene;
}
@@ -175,6 +149,7 @@ async function scrapeProfile(actor, entity, parameters) {
profile.url = `${entity.url}/models/${actor.id}`;
profile.description = actor.modelBio;
profile.gender = actor.gender;
if (actor.bio.about && !/\band\b/.test(actor.bio.about)) {
const bio = actor.bio.about.split(/\n/).filter(Boolean).reduce((acc, item) => {
@@ -235,15 +210,11 @@ async function scrapeProfile(actor, entity, parameters) {
}
async function fetchProfile(baseActor, { entity, parameters }) {
// const url = format(parameters.profiles, { slug: baseActor.slug });
const url = parameters.layout === 'organic'
? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json`
: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-modelscontent`}/_doc/${parameters.modelPrefix || ''}${baseActor.slug}`;
const url = `https://tours-store.psmcdn.net/${parameters.endpoint}/_doc/model_${baseActor.slug}`;
const res = await unprint.get(url);
if (res.ok && res.data) {
return scrapeProfile(parameters.layout === 'organic' ? res.data : res.data._source || res.body, entity, parameters);
return scrapeProfile(res.data._source || res.body, entity, parameters);
}
return res.status;
@@ -253,12 +224,4 @@ module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
organic: {
fetchLatest: fetchLatestOrganic,
fetchScene,
},
search: {
fetchLatest: fetchLatestSearch,
fetchScene,
},
};