DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Fixed Radical module, actively retrieving endpoint.

This commit is contained in:
DebaucheryLibrarian 2024-10-25 01:39:58 +02:00
parent 8b89119dfb
commit d7ff6cee08
1 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/scrapers/radical.js
View File

@ -67,20 +67,30 @@ function scrapeSceneApi(data, channel, parameters) {
release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss') || unprint.extractDate(data.formatted_date, 'Do MMM YYYY');
release.duration = data.seconds_duration || unprint.extractDuration(data.videos_duration);
release.actors = data.models_thumbs?.map((actor) => ({
// TWM in particular has a habit of putting two names in a single link https://tour.2girls1camera.com/scenes/richelle-ryan-ariella-ferrara
release.actors = (data.models_thumbs || data.models)?.flatMap((actor) => {
const actorNames = actor.name.split('&').map((actorName) => actorName.trim());
if (actorNames.length === 1) {
return {
name: actor.name,
avatar: actor.thumb,
})) || data.models;
};
}
release.poster = data.trailer_screencap;
return actorNames;
});
if (mime.getType(data.thumb) !== 'image/gif') {
release.teaser = data.thumb;
} else {
release.poster = data.trailer_screencap || data.thumb;
if (mime.getType(data.thumb) === 'image/gif') {
release.poster = [
release.poster,
data.trailer_screencap,
data.thumb,
];
} else {
release.poster = data.thumb;
// release.teaser = data.thumb;
}
release.photos = [
@ -94,6 +104,7 @@ function scrapeSceneApi(data, channel, parameters) {
release.caps = data.thumbs;
release.trailer = data.trailer_url;
release.teaser = data.special_thumbnails
?.filter((teaserUrl) => new URL(teaserUrl).pathname !== '/') // on Top Web Models, https://z7n5n3m8.ssl.hwcdn.net/ is listed as a teaser
.sort((teaserA, teaserB) => teaserOrder.findIndex((label) => teaserA.includes(label)) - teaserOrder.findIndex((label) => teaserB.includes(label)));
@ -168,10 +179,31 @@ function scrapeProfileApi(data, channel, scenes, parameters) {
return profile;
}
async function fetchEndpoint(channel, parameters) {
const res = await unprint.get(channel.url);
if (res.ok) {
const data = res.context.query.json('#__NEXT_DATA__');
if (data?.buildId) {
return data.buildId;
}
}
// still allow manual configuration as a back-up
return parameters.endpoint;
}
async function fetchLatestApi(channel, page, { parameters }) {
const endpoint = await fetchEndpoint(channel, parameters);
if (!endpoint) {
return null;
}
const url = parameters.site
? `${channel.parent.url}/_next/data/${parameters.endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
: `${channel.url}/_next/data/${parameters.endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
? `${channel.parent.url}/_next/data/${endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
: `${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
const res = await http.get(url);
@ -184,7 +216,8 @@ async function fetchLatestApi(channel, page, { parameters }) {
async function fetchSceneApi(url, channel, baseScene, { parameters }) {
const slug = new URL(url).pathname.split('/').at(-1);
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
if (res.ok && res.body.pageProps?.content) {
return scrapeSceneApi(res.body.pageProps.content, channel, parameters);
@ -194,7 +227,8 @@ async function fetchSceneApi(url, channel, baseScene, { parameters }) {
}
async function fetchProfileApi(actor, { channel, parameters }) {
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
if (res.ok && res.body.pageProps?.model) {
return scrapeProfileApi(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);