forked from DebaucheryLibrarian/traxxx
Moved PurgatoryX to Radical API scraper.
This commit is contained in:
parent
9c1124b97f
commit
3392b10182
|
|
@ -754,7 +754,6 @@ const networks = [
|
||||||
url: 'https://tour.topwebmodels.com',
|
url: 'https://tour.topwebmodels.com',
|
||||||
parent: 'radical',
|
parent: 'radical',
|
||||||
parameters: {
|
parameters: {
|
||||||
layout: 'api',
|
|
||||||
endpoint: '7D_7bQ7peaFQgjlOR42GH',
|
endpoint: '7D_7bQ7peaFQgjlOR42GH',
|
||||||
videos: 'scenes',
|
videos: 'scenes',
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -10004,6 +10004,9 @@ const sites = [
|
||||||
url: 'https://tour.purgatoryx.com',
|
url: 'https://tour.purgatoryx.com',
|
||||||
independent: true,
|
independent: true,
|
||||||
parent: 'radical',
|
parent: 'radical',
|
||||||
|
parameters: {
|
||||||
|
videos: 'episodes',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'Inserted',
|
name: 'Inserted',
|
||||||
|
|
@ -10012,9 +10015,9 @@ const sites = [
|
||||||
independent: true,
|
independent: true,
|
||||||
parent: 'radical',
|
parent: 'radical',
|
||||||
parameters: {
|
parameters: {
|
||||||
layout: 'api',
|
|
||||||
// endpoint: 'QrQe_TF3broC5P80XTIbd',
|
// endpoint: 'QrQe_TF3broC5P80XTIbd',
|
||||||
endpoint: 'E06HaYWbvDg4UG8fIsSi2',
|
endpoint: 'E06HaYWbvDg4UG8fIsSi2',
|
||||||
|
upcoming: 'upcoming',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
@ -10024,7 +10027,6 @@ const sites = [
|
||||||
independent: true,
|
independent: true,
|
||||||
parent: 'radical',
|
parent: 'radical',
|
||||||
parameters: {
|
parameters: {
|
||||||
layout: 'api',
|
|
||||||
// endpoint: 'nOpFJJgD_-c5PrBqecVXA',
|
// endpoint: 'nOpFJJgD_-c5PrBqecVXA',
|
||||||
endpoint: 'GUZuXniB1KVJwF9ZDWiRe',
|
endpoint: 'GUZuXniB1KVJwF9ZDWiRe',
|
||||||
},
|
},
|
||||||
|
|
@ -10036,7 +10038,6 @@ const sites = [
|
||||||
independent: true,
|
independent: true,
|
||||||
parent: 'radical',
|
parent: 'radical',
|
||||||
parameters: {
|
parameters: {
|
||||||
layout: 'api',
|
|
||||||
// endpoint: 'fnkMPhO2Gd-XwWTZHyftg',
|
// endpoint: 'fnkMPhO2Gd-XwWTZHyftg',
|
||||||
endpoint: 'K9_0ysd-cpgwtSaZ8_nPT',
|
endpoint: 'K9_0ysd-cpgwtSaZ8_nPT',
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,173 +0,0 @@
|
||||||
'use strict';
|
|
||||||
|
|
||||||
const qu = require('../utils/qu');
|
|
||||||
const http = require('../utils/http');
|
|
||||||
const slugify = require('../utils/slugify');
|
|
||||||
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
|
|
||||||
|
|
||||||
function scrapeAll(scenes) {
|
|
||||||
return scenes.map(({ query }) => {
|
|
||||||
const release = {};
|
|
||||||
|
|
||||||
release.title = query.cnt('.title');
|
|
||||||
release.url = query.url('.title a');
|
|
||||||
release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1];
|
|
||||||
|
|
||||||
release.date = query.date('.pub-date', 'MMM DD, YYYY');
|
|
||||||
release.duration = query.duration('.video-duration');
|
|
||||||
|
|
||||||
release.actors = query.all('.models a').map((el) => ({
|
|
||||||
name: query.cnt(el),
|
|
||||||
url: query.url(el, null),
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (query.exists('.thumb-big')) { // updates page
|
|
||||||
release.poster = query.img('.thumb-big', 'data-image') || JSON.parse(query.el('.thumbnails-wrap a', 'data-images'));
|
|
||||||
release.photos = [query.img('.thumb-top', 'data-image'), query.img('.thumb-bottom', 'data-image')];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (query.exists('.thumbnails-wrap')) { // actor page
|
|
||||||
try {
|
|
||||||
const images = JSON.parse(query.el('.thumbnails-wrap a', 'data-images'));
|
|
||||||
|
|
||||||
release.poster = images.slice(0, 1)[0];
|
|
||||||
release.photos = images.slice(1);
|
|
||||||
} catch (error) {
|
|
||||||
// images probably not available
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return release;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeUpcoming({ query }) {
|
|
||||||
const release = {};
|
|
||||||
|
|
||||||
release.url = query.url('.bottom-info a');
|
|
||||||
release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)?.[1];
|
|
||||||
release.title = query.cnt('.title');
|
|
||||||
|
|
||||||
release.actors = query.all('.model-wrap li').map((el) => ({
|
|
||||||
name: query.cnt(el, 'h5'),
|
|
||||||
url: query.url(el, '.model-thumb a'),
|
|
||||||
avatar: query.img(el, '.model-thumb img'),
|
|
||||||
}));
|
|
||||||
|
|
||||||
return release;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeScene({ query }, url) {
|
|
||||||
const release = {};
|
|
||||||
|
|
||||||
release.title = query.cnt('.title');
|
|
||||||
release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1];
|
|
||||||
release.date = query.date('.date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
|
||||||
|
|
||||||
release.description = query.cnt('.description p');
|
|
||||||
release.duration = query.duration('.total-time');
|
|
||||||
|
|
||||||
release.actors = query.all('.model-wrap li').map((el) => ({
|
|
||||||
name: query.cnt(el, 'h5'),
|
|
||||||
url: query.url(el, 'a'),
|
|
||||||
avatar: query.img(el),
|
|
||||||
}));
|
|
||||||
|
|
||||||
release.poster = query.poster();
|
|
||||||
release.photos = query.urls('.photos-slider a');
|
|
||||||
release.trailer = query.video();
|
|
||||||
|
|
||||||
release.comment = query.cnt('.series');
|
|
||||||
|
|
||||||
return release;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchLatest(channel, page) {
|
|
||||||
const res = await qu.getAll(`${channel.url}/episodes?page=${page}`, '.content-item');
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
return scrapeAll(res.items, channel);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchUpcoming(channel) {
|
|
||||||
const res = await qu.get(channel.url, '.upcoming-info-wrap');
|
|
||||||
|
|
||||||
if (res.ok && res.item) {
|
|
||||||
return [scrapeUpcoming(res.item, channel)];
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeProfile({ query }, url) {
|
|
||||||
const profile = { url };
|
|
||||||
|
|
||||||
const bio = Object.fromEntries(query.all('.model-desc li').map((el) => [slugify(query.cnt(el, 'span'), '_'), query.text(el)]));
|
|
||||||
|
|
||||||
profile.description = bio.bio;
|
|
||||||
|
|
||||||
profile.dateOfBirth = qu.extractDate(bio.birthdate, 'YYYY-MM-DD');
|
|
||||||
profile.birthPlace = bio.birthplace;
|
|
||||||
|
|
||||||
profile.hairColor = bio.hair_color;
|
|
||||||
profile.eyes = bio.eye_color;
|
|
||||||
|
|
||||||
profile.height = feetInchesToCm(bio.height);
|
|
||||||
profile.weight = lbsToKg(bio.weight);
|
|
||||||
profile.measurements = bio.measurements;
|
|
||||||
|
|
||||||
profile.avatar = query.img('.model-pic img');
|
|
||||||
|
|
||||||
profile.scenes = scrapeAll(qu.initAll(query.all('.content-item')));
|
|
||||||
|
|
||||||
return profile;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function searchActor(baseActor, channel) {
|
|
||||||
const searchRes = await http.post(`${channel.url}/search-preview`, { q: slugify(baseActor.name, ' ') }, {
|
|
||||||
encodeJSON: false,
|
|
||||||
headers: {
|
|
||||||
'Accept-Language': 'en-US,en;',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (searchRes.ok) {
|
|
||||||
const actorUrl = searchRes.body.find((item) => item.type === 'model' && slugify(item.title) === baseActor.slug)?.url;
|
|
||||||
|
|
||||||
return actorUrl || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchProfile(baseActor, context, include, retry = false) {
|
|
||||||
const actorUrl = (!retry && baseActor.url) || await searchActor(baseActor, context.entity);
|
|
||||||
|
|
||||||
if (!actorUrl) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = await qu.get(actorUrl);
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
return scrapeProfile(res.item, actorUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (baseActor.url) {
|
|
||||||
return fetchProfile(baseActor, context, include, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
fetchLatest,
|
|
||||||
fetchProfile,
|
|
||||||
fetchUpcoming,
|
|
||||||
scrapeAll,
|
|
||||||
scrapeScene,
|
|
||||||
deprecated: true,
|
|
||||||
};
|
|
||||||
|
|
@ -4,62 +4,28 @@ const unprint = require('unprint');
|
||||||
const mime = require('mime');
|
const mime = require('mime');
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
const qu = require('../utils/qu');
|
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
|
const { convert } = require('../utils/convert');
|
||||||
|
|
||||||
const teaserOrder = ['large', 'small', 'mobile', 'mp4', 'jpg'];
|
const teaserOrder = ['large', 'small', 'mobile', 'mp4', 'jpg'];
|
||||||
|
|
||||||
function scrapeSceneMetadata(data, channel) {
|
function getVideoPath(data, parameters) {
|
||||||
|
if (data.is_published === 0 && parameters.upcoming) {
|
||||||
|
return parameters.upcoming;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parameters.videos) {
|
||||||
|
return parameters.videos;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'videos';
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeScene(data, channel, parameters) {
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
release.entryId = data.id;
|
release.entryId = data.id;
|
||||||
release.url = `${channel.url}/tour/videos/${data.id}/${slugify(data.title, '-', { removePunctuation: true })}`;
|
release.url = `${channel.url}/${getVideoPath(data, parameters)}/${data.slug}`;
|
||||||
|
|
||||||
release.title = data.title;
|
|
||||||
release.description = data.description;
|
|
||||||
|
|
||||||
release.date = new Date(data.release_date);
|
|
||||||
release.duration = data.seconds_duration || qu.durationToSeconds(data.videos_duration);
|
|
||||||
|
|
||||||
release.actors = data.models.map((model) => ({
|
|
||||||
entryId: model.id,
|
|
||||||
name: model.name,
|
|
||||||
gender: model.gender,
|
|
||||||
avatar: model.thumb,
|
|
||||||
url: `${channel.url}/tour/models/${model.id}/${slugify(model.name, '-', { removePunctuation: true })}`,
|
|
||||||
}));
|
|
||||||
|
|
||||||
release.poster = data.trailer?.poster || [data.thumb?.replace('mobile.jpg', '.jpg'), data.thumb];
|
|
||||||
release.photos = [
|
|
||||||
data.extra_thumbs?.find((url) => /portrait1.jpg/.test(url)),
|
|
||||||
data.extra_thumbs?.find((url) => /scene.jpg/.test(url)),
|
|
||||||
data.extra_thumbs?.find((url) => /portrait2.jpg/.test(url)),
|
|
||||||
]; // ordered by chronology: portrait1.jpg and scene.jpg are usually pre-shoot poses, portrait2.jpg is the cumshot aftermath
|
|
||||||
|
|
||||||
release.trailer = data.trailer && {
|
|
||||||
src: data.trailer.src,
|
|
||||||
type: data.trailer.type,
|
|
||||||
};
|
|
||||||
|
|
||||||
release.teaser = data.special_thumbs;
|
|
||||||
|
|
||||||
release.tags = [].concat(data.tags?.map((tag) => tag.name));
|
|
||||||
release.qualities = data.downloads && Object.values(data.downloads)?.map((download) => download.meta_data.height);
|
|
||||||
release.stars = data.rating;
|
|
||||||
|
|
||||||
return release;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeAllMetadata(scenes, channel) {
|
|
||||||
return scenes.map((data) => scrapeSceneMetadata(data, channel));
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeSceneApi(data, channel, parameters) {
|
|
||||||
const release = {};
|
|
||||||
|
|
||||||
release.entryId = data.id;
|
|
||||||
release.url = `${channel.url}/${parameters.videos || 'videos'}/${data.slug}`;
|
|
||||||
|
|
||||||
release.title = data.title;
|
release.title = data.title;
|
||||||
release.description = data.description;
|
release.description = data.description;
|
||||||
|
|
@ -75,6 +41,7 @@ function scrapeSceneApi(data, channel, parameters) {
|
||||||
return {
|
return {
|
||||||
name: actor.name,
|
name: actor.name,
|
||||||
avatar: actor.thumb,
|
avatar: actor.thumb,
|
||||||
|
url: actor.slug && `${channel.url}/models/${actor.slug}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -119,64 +86,8 @@ function scrapeSceneApi(data, channel, parameters) {
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeAllApi(scenes, channel, parameters) {
|
function scrapeAll(scenes, channel, parameters) {
|
||||||
return scenes.map((data) => scrapeSceneApi(data, channel, parameters));
|
return scenes.map((data) => scrapeScene(data, channel, parameters));
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeProfileMetadata(data, channel) {
|
|
||||||
const profile = {};
|
|
||||||
|
|
||||||
profile.entryId = data.id;
|
|
||||||
profile.url = `${channel.url}/tour/models/${data.id}/${slugify(data.name, '-', { removePunctuation: true })}`;
|
|
||||||
|
|
||||||
profile.description = data.attributes.bio?.value;
|
|
||||||
profile.dateOfBirth = qu.parseDate(data.attributes.birthdate?.value, 'YYYY-MM-DD');
|
|
||||||
profile.gender = data.gender;
|
|
||||||
profile.age = data.attributes.age?.value;
|
|
||||||
profile.birthPlace = data.attributes.born?.value;
|
|
||||||
|
|
||||||
profile.measurements = data.attributes.measurements?.value;
|
|
||||||
profile.height = feetInchesToCm(data.attributes.height?.value);
|
|
||||||
profile.weight = lbsToKg(data.attributes.weight?.value);
|
|
||||||
|
|
||||||
profile.eyes = data.attributes.eyes?.value;
|
|
||||||
profile.hairColor = data.attributes.hair?.value;
|
|
||||||
|
|
||||||
profile.avatar = data.thumb;
|
|
||||||
profile.date = new Date(data.publish_date);
|
|
||||||
|
|
||||||
return profile;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeProfileApi(data, channel, scenes, parameters) {
|
|
||||||
const profile = {};
|
|
||||||
const bio = Object.fromEntries(Object.entries(data).map(([key, value]) => [key.toLowerCase(), value])); // keys are mixed upper and lowercase
|
|
||||||
|
|
||||||
profile.entryId = bio.id;
|
|
||||||
|
|
||||||
profile.description = bio.bio;
|
|
||||||
|
|
||||||
profile.gender = bio.gender;
|
|
||||||
|
|
||||||
profile.dateOfBirth = unprint.extractDate(bio.birthdate, 'YYYY-MM-DD');
|
|
||||||
profile.birthPlace = bio.born;
|
|
||||||
profile.age = bio.age;
|
|
||||||
|
|
||||||
profile.measurements = bio.measurements;
|
|
||||||
|
|
||||||
profile.height = feetInchesToCm(bio.height);
|
|
||||||
profile.weight = lbsToKg(bio.weight);
|
|
||||||
|
|
||||||
profile.eyes = bio.eyes;
|
|
||||||
profile.hairColor = bio.hair;
|
|
||||||
|
|
||||||
profile.avatar = data.thumb;
|
|
||||||
|
|
||||||
if (scenes) {
|
|
||||||
profile.scenes = scrapeAllApi(scenes, channel, parameters);
|
|
||||||
}
|
|
||||||
|
|
||||||
return profile;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchEndpoint(channel, parameters) {
|
async function fetchEndpoint(channel, parameters) {
|
||||||
|
|
@ -194,7 +105,7 @@ async function fetchEndpoint(channel, parameters) {
|
||||||
return parameters.endpoint;
|
return parameters.endpoint;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatestApi(channel, page, { parameters }) {
|
async function fetchLatest(channel, page, { parameters }) {
|
||||||
const endpoint = await fetchEndpoint(channel, parameters);
|
const endpoint = await fetchEndpoint(channel, parameters);
|
||||||
|
|
||||||
if (!endpoint) {
|
if (!endpoint) {
|
||||||
|
|
@ -208,86 +119,25 @@ async function fetchLatestApi(channel, page, { parameters }) {
|
||||||
const res = await http.get(url);
|
const res = await http.get(url);
|
||||||
|
|
||||||
if (res.ok && res.body.pageProps?.contents?.data) {
|
if (res.ok && res.body.pageProps?.contents?.data) {
|
||||||
return scrapeAllApi(res.body.pageProps.contents.data, channel, parameters);
|
return scrapeAll(res.body.pageProps.contents.data, channel, parameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchSceneApi(url, channel, baseScene, { parameters }) {
|
async function fetchUpcoming(channel, _page, { parameters }) {
|
||||||
const slug = new URL(url).pathname.split('/').at(-1);
|
const res = await unprint.get(channel.url, {
|
||||||
const endpoint = await fetchEndpoint(channel);
|
parser: {
|
||||||
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
|
|
||||||
|
|
||||||
if (res.ok && res.body.pageProps?.content) {
|
|
||||||
return scrapeSceneApi(res.body.pageProps.content, channel, parameters);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchProfileApi(actor, { channel, parameters }) {
|
|
||||||
const endpoint = await fetchEndpoint(channel);
|
|
||||||
const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
|
|
||||||
|
|
||||||
if (res.ok && res.body.pageProps?.model) {
|
|
||||||
return scrapeProfileApi(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchLatestMetadata(channel, page = 1) {
|
|
||||||
const url = `${channel.url}/tour/videos?page=${page}`;
|
|
||||||
const res = await http.get(url, {
|
|
||||||
parse: true,
|
|
||||||
extract: {
|
|
||||||
runScripts: 'dangerously',
|
runScripts: 'dangerously',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.ok && res.window.__DATA__) {
|
|
||||||
return scrapeAllMetadata(res.window.__DATA__.videos.items, channel);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
return res.window.__DATA__?.error || null;
|
const data = res.context.query.json('#__NEXT_DATA__');
|
||||||
}
|
const scene = data?.props.pageProps.upcoming_scene;
|
||||||
|
|
||||||
return res.status;
|
if (scene) {
|
||||||
}
|
return scrapeScene(scene, channel, parameters);
|
||||||
|
|
||||||
async function fetchSceneMetadata(url, channel) {
|
|
||||||
const res = await http.get(url, {
|
|
||||||
parse: true,
|
|
||||||
extract: {
|
|
||||||
runScripts: 'dangerously',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (res.ok && res.window.__DATA__?.video) {
|
|
||||||
return scrapeSceneMetadata(res.window.__DATA__.video, channel);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
return res.window.__DATA__?.error || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchProfileMetadata(actor, channel) {
|
|
||||||
const res = await http.get(`${channel.url}/tour/search-preview/${actor.name}`, {
|
|
||||||
headers: {
|
|
||||||
'X-Requested-With': 'XMLHttpRequest',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
const model = res.body.models?.items.find((modelX) => slugify(modelX.name) === actor.slug);
|
|
||||||
|
|
||||||
if (model) {
|
|
||||||
return scrapeProfileMetadata(model, channel);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -296,16 +146,63 @@ async function fetchProfileMetadata(actor, channel) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function fetchScene(url, channel, _baseScene, { parameters }) {
|
||||||
|
const slug = new URL(url).pathname.split('/').at(-1);
|
||||||
|
const endpoint = await fetchEndpoint(channel);
|
||||||
|
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
|
||||||
|
|
||||||
|
if (res.ok && res.body.pageProps?.content) {
|
||||||
|
return scrapeScene(res.body.pageProps.content, channel, parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeProfile(data, channel, scenes, parameters) {
|
||||||
|
const profile = {};
|
||||||
|
const bio = Object.fromEntries(Object.entries(data).map(([key, value]) => [slugify(key, '_'), value])); // keys are mixed upper and lowercase
|
||||||
|
|
||||||
|
profile.entryId = bio.id;
|
||||||
|
|
||||||
|
profile.description = bio.bio;
|
||||||
|
|
||||||
|
profile.gender = bio.gender;
|
||||||
|
|
||||||
|
profile.dateOfBirth = unprint.extractDate(bio.birthdate, 'YYYY-MM-DD');
|
||||||
|
profile.birthPlace = bio.born || bio.birthplace;
|
||||||
|
profile.age = bio.age;
|
||||||
|
|
||||||
|
profile.measurements = bio.measurements;
|
||||||
|
|
||||||
|
profile.height = convert(bio.height, 'cm');
|
||||||
|
profile.weight = convert(bio.weight, 'lb', 'kg');
|
||||||
|
|
||||||
|
profile.eyes = bio.eyes || bio.eye_color;
|
||||||
|
profile.hairColor = bio.hair || bio.hair_color;
|
||||||
|
|
||||||
|
profile.avatar = data.thumb;
|
||||||
|
|
||||||
|
if (scenes) {
|
||||||
|
profile.scenes = scrapeAll(scenes, channel, parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchProfile(actor, { channel, parameters }) {
|
||||||
|
const endpoint = await fetchEndpoint(channel);
|
||||||
|
const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`);
|
||||||
|
|
||||||
|
if (res.ok && res.body.pageProps?.model) {
|
||||||
|
return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
metadata: {
|
fetchLatest,
|
||||||
// probably deprecated
|
fetchUpcoming,
|
||||||
fetchLatest: fetchLatestMetadata,
|
fetchScene,
|
||||||
fetchScene: fetchSceneMetadata,
|
fetchProfile,
|
||||||
fetchProfile: fetchProfileMetadata,
|
|
||||||
},
|
|
||||||
api: {
|
|
||||||
fetchLatest: fetchLatestApi,
|
|
||||||
fetchScene: fetchSceneApi,
|
|
||||||
fetchProfile: fetchProfileApi,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,6 @@ const porncz = require('./porncz');
|
||||||
const pornhub = require('./pornhub');
|
const pornhub = require('./pornhub');
|
||||||
const pornworld = require('./pornworld');
|
const pornworld = require('./pornworld');
|
||||||
const privateNetwork = require('./private'); // reserved keyword
|
const privateNetwork = require('./private'); // reserved keyword
|
||||||
const purgatoryx = require('./purgatoryx');
|
|
||||||
const radical = require('./radical');
|
const radical = require('./radical');
|
||||||
const rickysroom = require('./rickysroom');
|
const rickysroom = require('./rickysroom');
|
||||||
const sexlikereal = require('./sexlikereal');
|
const sexlikereal = require('./sexlikereal');
|
||||||
|
|
@ -169,7 +168,6 @@ const scrapers = {
|
||||||
pornpros: whalemember,
|
pornpros: whalemember,
|
||||||
pornplus: whalemember,
|
pornplus: whalemember,
|
||||||
private: privateNetwork,
|
private: privateNetwork,
|
||||||
purgatoryx,
|
|
||||||
radical,
|
radical,
|
||||||
rickysroom,
|
rickysroom,
|
||||||
sayuncle: teamskeet,
|
sayuncle: teamskeet,
|
||||||
|
|
@ -226,6 +224,8 @@ const scrapers = {
|
||||||
twistys: aylo,
|
twistys: aylo,
|
||||||
// gamma
|
// gamma
|
||||||
'21sextury': gamma,
|
'21sextury': gamma,
|
||||||
|
'3rddegreefilms': gamma,
|
||||||
|
addicted2girls: gamma,
|
||||||
biphoria: gamma,
|
biphoria: gamma,
|
||||||
blakemason: gamma,
|
blakemason: gamma,
|
||||||
blowpass: gamma,
|
blowpass: gamma,
|
||||||
|
|
@ -239,6 +239,7 @@ const scrapers = {
|
||||||
fantasymassage: gamma,
|
fantasymassage: gamma,
|
||||||
filthykings: gamma,
|
filthykings: gamma,
|
||||||
gangbangcreampie: gamma,
|
gangbangcreampie: gamma,
|
||||||
|
genderxfilms: gamma,
|
||||||
girlsway: gamma,
|
girlsway: gamma,
|
||||||
gloryholesecrets: gamma,
|
gloryholesecrets: gamma,
|
||||||
peternorth: gamma,
|
peternorth: gamma,
|
||||||
|
|
@ -253,20 +254,30 @@ const scrapers = {
|
||||||
wicked: gamma,
|
wicked: gamma,
|
||||||
xempire: gamma,
|
xempire: gamma,
|
||||||
zerotolerancefilms: gamma,
|
zerotolerancefilms: gamma,
|
||||||
'3rddegreefilms': gamma,
|
|
||||||
addicted2girls: gamma,
|
|
||||||
genderxfilms: gamma,
|
|
||||||
// mike adriano
|
// mike adriano
|
||||||
trueanal: mikeadriano,
|
|
||||||
swallowed: mikeadriano,
|
|
||||||
nympho: mikeadriano,
|
|
||||||
dirtyauditions: mikeadriano,
|
|
||||||
analonly: mikeadriano,
|
|
||||||
allanal: mikeadriano,
|
allanal: mikeadriano,
|
||||||
// the flourish
|
analonly: mikeadriano,
|
||||||
theflourishxxx: theflourish,
|
dirtyauditions: mikeadriano,
|
||||||
|
nympho: mikeadriano,
|
||||||
|
swallowed: mikeadriano,
|
||||||
|
trueanal: mikeadriano,
|
||||||
|
// radical
|
||||||
|
bjraw: radical,
|
||||||
|
gotfilled: radical,
|
||||||
|
inserted: radical,
|
||||||
|
purgatoryx: radical,
|
||||||
|
topwebmodels: radical,
|
||||||
|
// hush / hussiepass
|
||||||
|
eyeontheguy: hush,
|
||||||
|
hushpass: hush,
|
||||||
|
hussiepass: hush,
|
||||||
|
interracialpass: hush,
|
||||||
|
interracialpovs: hush,
|
||||||
|
povpornstars: hush,
|
||||||
|
seehimfuck: hush,
|
||||||
// etc
|
// etc
|
||||||
'18vr': badoink,
|
'18vr': badoink,
|
||||||
|
theflourishxxx: theflourish,
|
||||||
adultempire,
|
adultempire,
|
||||||
archangel,
|
archangel,
|
||||||
allherluv: missax,
|
allherluv: missax,
|
||||||
|
|
@ -286,7 +297,6 @@ const scrapers = {
|
||||||
badoinkvr: badoink,
|
badoinkvr: badoink,
|
||||||
bamvisions,
|
bamvisions,
|
||||||
bang,
|
bang,
|
||||||
bjraw: radical,
|
|
||||||
bluedonkeymedia,
|
bluedonkeymedia,
|
||||||
delphine: modelmedia,
|
delphine: modelmedia,
|
||||||
meidenvanholland: bluedonkeymedia,
|
meidenvanholland: bluedonkeymedia,
|
||||||
|
|
@ -301,22 +311,15 @@ const scrapers = {
|
||||||
doubleviewcasting: firstanalquest,
|
doubleviewcasting: firstanalquest,
|
||||||
dtfsluts: fullpornnetwork,
|
dtfsluts: fullpornnetwork,
|
||||||
exploitedx, // only from known URL that will specify site
|
exploitedx, // only from known URL that will specify site
|
||||||
eyeontheguy: hush,
|
|
||||||
firstanalquest,
|
firstanalquest,
|
||||||
forbondage: porndoe,
|
forbondage: porndoe,
|
||||||
freeones,
|
freeones,
|
||||||
girlfaction: fullpornnetwork,
|
girlfaction: fullpornnetwork,
|
||||||
gotfilled: radical,
|
|
||||||
hergape: fullpornnetwork,
|
hergape: fullpornnetwork,
|
||||||
hitzefrei,
|
hitzefrei,
|
||||||
homemadeanalwhores: fullpornnetwork,
|
homemadeanalwhores: fullpornnetwork,
|
||||||
hookuphotshot,
|
hookuphotshot,
|
||||||
hotcrazymess: nubiles,
|
hotcrazymess: nubiles,
|
||||||
hushpass: hush,
|
|
||||||
hussiepass: hush,
|
|
||||||
inserted: radical,
|
|
||||||
interracialpass: hush,
|
|
||||||
interracialpovs: hush,
|
|
||||||
inthecrack,
|
inthecrack,
|
||||||
jamesdeen: fullpornnetwork,
|
jamesdeen: fullpornnetwork,
|
||||||
jerkaoke: modelmedia,
|
jerkaoke: modelmedia,
|
||||||
|
|
@ -357,14 +360,11 @@ const scrapers = {
|
||||||
pornhub,
|
pornhub,
|
||||||
pornworld,
|
pornworld,
|
||||||
povperverts: fullpornnetwork,
|
povperverts: fullpornnetwork,
|
||||||
povpornstars: hush,
|
|
||||||
private: privateNetwork,
|
private: privateNetwork,
|
||||||
purgatoryx,
|
|
||||||
realvr: badoink,
|
realvr: badoink,
|
||||||
rickysroom,
|
rickysroom,
|
||||||
sayuncle: teamskeet,
|
sayuncle: teamskeet,
|
||||||
score,
|
score,
|
||||||
seehimfuck: hush,
|
|
||||||
sexlikereal,
|
sexlikereal,
|
||||||
spermmania: snowvalley,
|
spermmania: snowvalley,
|
||||||
handjobjapan: snowvalley,
|
handjobjapan: snowvalley,
|
||||||
|
|
|
||||||
|
|
@ -113,11 +113,16 @@ const actors = [
|
||||||
// perv city
|
// perv city
|
||||||
{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
{ entity: 'pervcity', name: 'Brooklyn Gray', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
{ entity: 'dpdiva', name: 'Liz Jordan', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
|
// radical
|
||||||
|
{ entity: 'bjraw', name: 'Nikki Knightly', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
|
{ entity: 'gotfilled', name: 'Alexa Chains', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
|
{ entity: 'inserted', name: 'Anissa Kate', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
|
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
|
{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
|
||||||
// etc.
|
// etc.
|
||||||
{ entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] },
|
{ entity: 'archangel', name: 'Summer Brielle', fields: ['avatar', 'description', 'dateOfBirth', 'age', 'measurements', 'height', 'aliases'] },
|
||||||
{ entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] },
|
{ entity: 'theflourishxxx', name: 'XWifeKaren', fields: ['avatar', 'description'] },
|
||||||
{ entity: 'hookuphotshot', name: 'Kenzie Reeves', fields: ['avatar', 'description'] },
|
{ entity: 'hookuphotshot', name: 'Kenzie Reeves', fields: ['avatar', 'description'] },
|
||||||
{ entity: 'inserted', name: 'Kenzie Reeves', fields: ['avatar', 'description'] },
|
|
||||||
];
|
];
|
||||||
|
|
||||||
const actorScrapers = scrapers.actors;
|
const actorScrapers = scrapers.actors;
|
||||||
|
|
@ -153,7 +158,7 @@ const validators = {
|
||||||
nationality: (value) => typeof value === 'string' && value.length > 3,
|
nationality: (value) => typeof value === 'string' && value.length > 3,
|
||||||
// height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), // ft in needs to be converted
|
// height: (value) => !!Number(value) || /\d'\d{1,2}"/.test(value), // ft in needs to be converted
|
||||||
height: (value) => !!Number(value) && value > 150,
|
height: (value) => !!Number(value) && value > 150,
|
||||||
weight: (value) => !!Number(value) && value > 50,
|
weight: (value) => !!Number(value) && value > 40,
|
||||||
eyes: (value) => typeof value === 'string' && value.length > 3,
|
eyes: (value) => typeof value === 'string' && value.length > 3,
|
||||||
hairColor: (value) => typeof value === 'string' && value.length > 3,
|
hairColor: (value) => typeof value === 'string' && value.length > 3,
|
||||||
measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module
|
measurements: (value) => /(\d+)([a-z]+)?(?:\s*[-x]\s*(\d+)\s*[-x]\s*(\d+))?/i.test(value), // from actors module
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue