DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Fixed Score scraper breaking on scene with homepage URL. Updated Say Uncle Team Skeet endpoint.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-01-15 23:18:28 +01:00
parent e2124acad2
commit b7d58cbe13
4 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/scrapers/score.js
View File

@@ -27,6 +27,7 @@ function scrapeAll(scenes, channel, parameters) {
const poster = query.img('.item-img img');
const url = stripQuery(query.url('a.i-title, .item-img a'));
const { pathname, hostname } = new URL(url);
release.title = query.content('a.i-title, h2.i-title');
release.duration = query.duration('.time-ol');
@@ -44,7 +45,7 @@ function scrapeAll(scenes, channel, parameters) {
release.actors = query.content('.i-model').split(',').map((actor) => actor.trim());
if (url.includes('join.') || url.includes('/join')) {
if (hostname.includes('join.') || pathname.includes('/join') || pathname.length <= 1) {
// no link available, attempt to reconstruct from poster URL
const entryId = poster?.match(/posting_(\d+)/)?.[1];
@@ -58,7 +59,7 @@ function scrapeAll(scenes, channel, parameters) {
}
} else {
release.url = url;
release.entryId = new URL(release.url).pathname.match(/\/(\d+)\/?$/)[1];
release.entryId = pathname.match(/\/(\d+)\/?$/)[1];
}
if (poster) {

8
src/scrapers/teamskeet.js
View File

@@ -128,7 +128,7 @@ function scrapeProfile(actor, entity, parameters) {
}
async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.endpoint}-videoscontent/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
@@ -157,7 +157,7 @@ async function fetchLatestOrganic(channel, page, context) {
}
async function fetchLatestSearch(channel, page = 1, { parameters }) {
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:%22${parameters.id}%22%20AND%20type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:%22${parameters.id}%22%20AND%20type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
if (res.ok) {
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
@@ -177,7 +177,7 @@ async function fetchScene(url, channel, baseScene, { parameters }) {
const res = await http.get({
organic: `https://store.psmcdn.net/${parameters.endpoint}/moviesContent/${sceneSlug}.json`,
search: `https://tours-store.psmcdn.net/ts_network/_search/?q=(id:${sceneSlug})&size=1`,
undefined: `https://tours-store.psmcdn.net/${parameters.endpoint}-videoscontent/_doc/${sceneSlug}`,
undefined: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_doc/${sceneSlug}`,
}[parameters.layout]);
if (res.ok && res.body.found) {
@@ -195,7 +195,7 @@ async function fetchProfile(baseActor, { entity, parameters }) {
// const url = format(parameters.profiles, { slug: baseActor.slug });
const url = parameters.layout === 'organic'
? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json`
: `https://tours-store.psmcdn.net/${parameters.endpoint}-modelscontent/_doc/${baseActor.slug}`;
: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-modelscontent`}/_doc/${parameters.modelPrefix || ''}${baseActor.slug}`;
const res = await qu.get(url);