DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated Radical to support Top Web Models.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2023-08-16 06:31:49 +02:00
parent 46c514f530
commit c94dcdd9e6
4 changed files with 54 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/scrapers/radical.js
View File

@@ -54,11 +54,11 @@ function scrapeAllMetadata(scenes, channel) {
return scenes.map((data) => scrapeSceneMetadata(data, channel));
}
function scrapeSceneApi(data, channel) {
function scrapeSceneApi(data, channel, parameters) {
const release = {};
release.entryId = data.id;
release.url = `${channel.url}/videos/${data.slug}`;
release.url = `${channel.url}/${parameters.videos || 'videos'}/${data.slug}`;
release.title = data.title;
release.description = data.description;
@@ -71,19 +71,22 @@ function scrapeSceneApi(data, channel) {
avatar: actor.thumb,
})) || data.models;
release.poster = data.trailer_screencap;
release.poster = data.trailer_screencap || data.thumb;
release.photos = [
...data.previews?.full || [],
...data.extra_thumbnails?.filter((thumbnail) => !thumbnail.includes('mobile') // mobile is the cropped photo of a photo already in the set
&& !(thumbnail.includes('_scene') && release.poster?.includes('_scene')) // likely the same photo, filename may differ so cannot compare full path
&& !(thumbnail.includes('_player') && release.poster?.includes('_player'))) || [],
&& !(thumbnail.includes('_player') && release.poster?.includes('_player'))
&& !(thumbnail.includes('1920') && release.poster?.includes('1920'))) || [],
];
release.caps = data.thumbs;
release.trailer = data.trailer_url;
release.teaser = data.special_thumbnails.sort((teaserA, teaserB) => teaserOrder.findIndex((label) => teaserA.includes(label)) - teaserOrder.findIndex((label) => teaserB.includes(label)));
release.teaser = data.special_thumbnails
?.filter((teaserUrl) => new URL(teaserUrl).pathname !== '/') // on Top Web Models, https://z7n5n3m8.ssl.hwcdn.net/ is listed as a teaser
.sort((teaserA, teaserB) => teaserOrder.findIndex((label) => teaserA.includes(label)) - teaserOrder.findIndex((label) => teaserB.includes(label)));
release.tags = data.tags;
@@ -95,8 +98,8 @@ function scrapeSceneApi(data, channel) {
return release;
}
function scrapeAllApi(scenes, channel) {
return scenes.map((data) => scrapeSceneApi(data, channel));
function scrapeAllApi(scenes, channel, parameters) {
return scenes.map((data) => scrapeSceneApi(data, channel, parameters));
}
function scrapeProfileMetadata(data, channel) {
@@ -124,7 +127,7 @@ function scrapeProfileMetadata(data, channel) {
return profile;
}
function scrapeProfileApi(data, channel, scenes) {
function scrapeProfileApi(data, channel, scenes, parameters) {
const profile = {};
const bio = Object.fromEntries(Object.entries(data).map(([key, value]) => [key.toLowerCase(), value])); // keys are mixed upper and lowercase
@@ -149,17 +152,21 @@ function scrapeProfileApi(data, channel, scenes) {
profile.avatar = data.thumb;
if (scenes) {
profile.scenes = scrapeAllApi(scenes, channel);
profile.scenes = scrapeAllApi(scenes, channel, parameters);
}
return profile;
}
async function fetchLatestApi(channel, page, { parameters }) {
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/videos.json?order_by=publish_date&sort_by=desc&per_page=8&page=${page}`);
const url = parameters.site
? `${channel.parent.url}/_next/data/${parameters.endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
: `${channel.url}/_next/data/${parameters.endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
const res = await http.get(url);
if (res.ok && res.body.pageProps?.contents?.data) {
return scrapeAllApi(res.body.pageProps.contents.data, channel);
return scrapeAllApi(res.body.pageProps.contents.data, channel, parameters);
}
return res.status;
@@ -167,10 +174,10 @@ async function fetchLatestApi(channel, page, { parameters }) {
async function fetchSceneApi(url, channel, baseScene, { parameters }) {
const slug = new URL(url).pathname.split('/').at(-1);
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/videos/${slug}.json?slug=${slug}`);
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
if (res.ok && res.body.pageProps?.content) {
return scrapeSceneApi(res.body.pageProps.content, channel);
return scrapeSceneApi(res.body.pageProps.content, channel, parameters);
}
return res.status;
@@ -180,7 +187,7 @@ async function fetchProfileApi(actor, { channel, parameters }) {
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
if (res.ok && res.body.pageProps?.model) {
return scrapeProfileApi(res.body.pageProps.model, channel, res.body.pageProps.model_contents);
return scrapeProfileApi(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
}
return res.status;