traxxx/src/scrapers/purgatoryx.js

173 lines
4.3 KiB
JavaScript
Raw Normal View History

2021-11-27 22:55:16 +00:00
'use strict';
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
function scrapeAll(scenes) {
return scenes.map(({ query }) => {
const release = {};
release.title = query.cnt('.title');
release.url = query.url('.title a');
release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1];
release.date = query.date('.pub-date', 'MMM DD, YYYY');
release.duration = query.duration('.video-duration');
release.actors = query.all('.models a').map((el) => ({
name: query.cnt(el),
url: query.url(el, null),
}));
if (query.exists('.thumb-big')) { // updates page
release.poster = query.img('.thumb-big', 'data-image') || JSON.parse(query.el('.thumbnails-wrap a', 'data-images'));
release.photos = [query.img('.thumb-top', 'data-image'), query.img('.thumb-bottom', 'data-image')];
}
if (query.exists('.thumbnails-wrap')) { // actor page
try {
const images = JSON.parse(query.el('.thumbnails-wrap a', 'data-images'));
release.poster = images.slice(0, 1)[0];
release.photos = images.slice(1);
} catch (error) {
// images probably not available
}
}
return release;
});
}
2021-11-28 00:20:39 +00:00
function scrapeUpcoming({ query }) {
const release = {};
release.url = query.url('.bottom-info a');
release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)?.[1];
release.title = query.cnt('.title');
release.actors = query.all('.model-wrap li').map((el) => ({
name: query.cnt(el, 'h5'),
url: query.url(el, '.model-thumb a'),
avatar: query.img(el, '.model-thumb img'),
}));
return release;
}
2021-11-27 22:55:16 +00:00
function scrapeScene({ query }, url) {
const release = {};
release.title = query.cnt('.title');
release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1];
release.date = query.date('.date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
release.description = query.cnt('.description p');
release.duration = query.duration('.total-time');
release.actors = query.all('.model-wrap li').map((el) => ({
name: query.cnt(el, 'h5'),
url: query.url(el, 'a'),
avatar: query.img(el),
}));
release.poster = query.poster();
release.photos = query.urls('.photos-slider a');
release.trailer = query.video();
release.comment = query.cnt('.series');
return release;
}
async function fetchLatest(channel, page) {
const res = await qu.getAll(`${channel.url}/episodes?page=${page}`, '.content-item');
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
2021-11-28 00:20:39 +00:00
async function fetchUpcoming(channel) {
const res = await qu.get(channel.url, '.upcoming-info-wrap');
if (res.ok && res.item) {
return [scrapeUpcoming(res.item, channel)];
}
return res.status;
}
2021-11-27 22:55:16 +00:00
function scrapeProfile({ query }, url) {
const profile = { url };
const bio = Object.fromEntries(query.all('.model-desc li').map((el) => [slugify(query.cnt(el, 'span'), '_'), query.text(el)]));
profile.description = bio.bio;
profile.dateOfBirth = qu.extractDate(bio.birthdate, 'YYYY-MM-DD');
profile.birthPlace = bio.birthplace;
profile.hairColor = bio.hair_color;
profile.eyes = bio.eye_color;
profile.height = feetInchesToCm(bio.height);
profile.weight = lbsToKg(bio.weight);
profile.measurements = bio.measurements;
profile.avatar = query.img('.model-pic img');
profile.scenes = scrapeAll(qu.initAll(query.all('.content-item')));
return profile;
}
async function searchActor(baseActor, channel) {
const searchRes = await http.post(`${channel.url}/search-preview`, { q: slugify(baseActor.name, ' ') }, {
encodeJSON: false,
headers: {
'Accept-Language': 'en-US,en;',
},
});
if (searchRes.ok) {
const actorUrl = searchRes.body.find((item) => item.type === 'model' && slugify(item.title) === baseActor.slug)?.url;
return actorUrl || null;
}
return null;
}
async function fetchProfile(baseActor, context, include, retry = false) {
const actorUrl = (!retry && baseActor.url) || await searchActor(baseActor, context.entity);
if (!actorUrl) {
return null;
}
const res = await qu.get(actorUrl);
if (res.ok) {
return scrapeProfile(res.item, actorUrl);
}
if (baseActor.url) {
return fetchProfile(baseActor, context, include, true);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchProfile,
2021-11-28 00:20:39 +00:00
fetchUpcoming,
2021-11-27 22:55:16 +00:00
scrapeAll,
scrapeScene,
};