DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Added profile scraper tests (WIP), fixed some profile scrapers. Fixed slugify not breaking existing slugs.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-01-10 02:58:50 +01:00
parent 5acc2c607b
commit bddc33a734
12 changed files with 293 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/scrapers/spizoo.js
View File

@@ -31,12 +31,26 @@ function scrapeAll(scenes) {
});
}
async function fetchLatest(channel, page) {
const res = await unprint.get(`${channel.url}${format(channel.parameters?.latest || '/categories/movies_{page}_d.html', { page })}`, {
selectAll: '.thumb-big, .thumb-video, .thumbnail, .thumbnail-popular, .full-thumbnail',
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
function scrapeScene({ query }, { url, entity }) {
const release = {};
release.entryId = getEntryId(url);
release.title = query.content(['#media-holder .title', '.content-holder h1', '#scene h1', 'h2.titular', 'title'])?.replace(/\s+-$/, '');
console.log(release);
release.date = query.date('#sceneInfo .date, #trailer-data .date', 'YYYY-MM-DD');
release.duration = query.duration('#sceneInfo .data-others, #trailer-data', /\d+:\d+/);
@@ -67,6 +81,28 @@ function scrapeScene({ query }, { url, entity }) {
return release;
}
function stripSizeParams(source) {
if (!source) {
return [];
}
try {
const url = new URL(source);
const params = url.searchParams;
params.delete('imgh');
params.delete('imgw');
params.delete('imgq');
return [
`${url.origin}${url.pathname}?${params.toString()}`,
source,
];
} catch (error) {
return [];
}
}
function scrapeProfile({ query }) {
const profile = {};
const bioKeys = query.contents('.statsText b');
@@ -77,13 +113,14 @@ function scrapeProfile({ query }) {
[slugify(key, '_')]: bioValues[index],
}), {});
profile.description = query.contents('.descriptionText');
profile.description = query.content('.descriptionText');
profile.avatar = [
...stripSizeParams(query.img('.model-bio-pic img', { attribute: 'src' })), // not available on e.g. Raw Attack
query.img('.model-bio-pic img', { attribute: 'src0_3x' }),
query.img('.model-bio-pic img', { attribute: 'src0_2x' }),
query.img('.model-bio-pic img', { attribute: 'src0_3x' }), // unnecessarily big
query.img('.model-bio-pic img', { attribute: 'src0_1x' }),
];
].filter(Boolean);
profile.height = Number(bio.height?.match(/(\d+)\s?cm/i)?.[1]);
profile.dateOfBirth = unprint.extractDate(bio.date_of_birth, 'MMMM D, YYYY');
@@ -108,18 +145,6 @@ function scrapeProfile({ query }) {
return profile;
}
async function fetchLatest(channel, page) {
const res = await unprint.get(`${channel.url}${format(channel.parameters?.latest || '/categories/movies_{page}_d.html', { page })}`, {
selectAll: '.thumb-big, .thumb-video, .thumbnail, .thumbnail-popular, .full-thumbnail',
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchProfile(actor, channel) {
if (actor.url) {
const res = await unprint.get(actor.url);