boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Handling inconsistent album data in Ricky's Room scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-07-26 17:59:37 +02:00
parent d21ba65a43
commit 5f0320fde0
1 changed files with 31 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/scrapers/rickysroom.js
View File

@@ -23,9 +23,13 @@ function scrapeScene(data, channel) {
release.tags = data.tags;
release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
release.photos = data.previews?.full
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
.concat(data.thumbs); // screenshots
// inconsistent data structure. if there is an album, it's previews: { full: [...] }, if it's empty, it's previews: []
if (Array.isArray(data.previews?.full)) {
release.photos = data.previews?.full
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
.concat(data.thumbs); // screenshots
}
release.trailer = data.trailer_url;
release.teaser = data.special_thumbnails;
@@ -36,30 +40,6 @@ function scrapeScene(data, channel) {
return release;
}
function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
profile.weight = lbsToKg(data.Weight || data.weight);
profile.eyes = data.Eyes || data.eyes;
profile.hairColor = data.Hair || data.hair;
profile.avatar = data.thumb;
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
return profile;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
const res = await qu.get(url);
@@ -102,6 +82,30 @@ async function fetchScene(url, channel, baseRelease) {
return res.status;
}
function scrapeProfile(data, scenes, entity) {
const profile = {};
profile.entryId = data.id;
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
profile.description = data.Bio || data.bio;
profile.birthPlace = data.Born || data.born;
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
profile.measurements = data.Measurements || data.Measurements;
profile.height = feetInchesToCm(data.Height || data.height);
profile.weight = lbsToKg(data.Weight || data.weight);
profile.eyes = data.Eyes || data.eyes;
profile.hairColor = data.Hair || data.hair;
profile.avatar = data.thumb;
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
return profile;
}
async function fetchProfile({ slug }, entity) {
const url = `${entity.url}/models/${slug}`;
const res = await qu.get(url);