Refactored Whale Member (Porn Pros) scraper.
This commit is contained in:
DebaucheryLibrarian
parent
221203d889
commit
4d91060fb8
File diff suppressed because it is too large
Load Diff
|
|
@ -132,6 +132,8 @@
|
|||
"object-merge-advanced": "^12.1.0",
|
||||
"object.omit": "^3.0.0",
|
||||
"pg": "^8.11.3",
|
||||
"playwright": "^1.57.0",
|
||||
"playwright-core": "^1.57.0",
|
||||
"postgraphile": "^4.14.0",
|
||||
"postgraphile-plugin-connection-filter": "^2.3.0",
|
||||
"promise-task-queue": "^1.2.0",
|
||||
|
|
@ -149,7 +151,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.16.3",
|
||||
"unprint": "^0.17.4",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
|
|
|||
|
|
@ -746,6 +746,10 @@ const tags = [
|
|||
name: 'leather',
|
||||
slug: 'leather',
|
||||
},
|
||||
{
|
||||
name: 'leg warmers',
|
||||
slug: 'leg-warmers',
|
||||
},
|
||||
{
|
||||
name: 'lesbian',
|
||||
slug: 'lesbian',
|
||||
|
|
|
|||
|
|
@ -638,9 +638,6 @@ const networks = [
|
|||
name: 'Porn+',
|
||||
url: 'https://pornplus.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
layout: 'plus',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'private',
|
||||
|
|
|
|||
|
|
@ -9319,21 +9319,21 @@ const sites = [
|
|||
{
|
||||
name: 'Porn Pros',
|
||||
slug: 'pornpros',
|
||||
url: 'https://pornpros.com/site/pornpros',
|
||||
url: 'https://pornpros.com/sites/pornpros',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Real Ex Girlfriends',
|
||||
slug: 'realexgirlfriends',
|
||||
alias: ['reg'],
|
||||
url: 'https://pornpros.com/site/realexgirlfriends',
|
||||
url: 'https://pornpros.com/sites/realexgirlfriends',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: '18 Years Old',
|
||||
slug: 'eighteenyearsold',
|
||||
alias: ['18yo'],
|
||||
url: 'https://pornpros.com/site/18yearsold',
|
||||
url: 'https://pornpros.com/sites/18yearsold',
|
||||
tags: ['teen'],
|
||||
parent: 'pornpros',
|
||||
},
|
||||
|
|
@ -9341,14 +9341,14 @@ const sites = [
|
|||
name: 'Massage Creep',
|
||||
slug: 'massagecreep',
|
||||
alias: ['mc'],
|
||||
url: 'https://pornpros.com/site/massagecreep',
|
||||
url: 'https://pornpros.com/sites/massagecreep',
|
||||
tags: ['massage'],
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Deep Throat Love',
|
||||
slug: 'deepthroatlove',
|
||||
url: 'https://pornpros.com/site/deepthroatlove',
|
||||
url: 'https://pornpros.com/sites/deepthroatlove',
|
||||
tags: ['blowjob', 'deepthroat'],
|
||||
parent: 'pornpros',
|
||||
},
|
||||
|
|
@ -9356,74 +9356,74 @@ const sites = [
|
|||
name: 'Teen BFF',
|
||||
slug: 'teenbff',
|
||||
alias: ['tbff'],
|
||||
url: 'https://pornpros.com/site/teenbff',
|
||||
url: 'https://pornpros.com/sites/teenbff',
|
||||
tags: ['mff'],
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Shady P.I.',
|
||||
slug: 'shadypi',
|
||||
url: 'https://pornpros.com/site/shadypi',
|
||||
url: 'https://pornpros.com/sites/shadypi',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Cruelty Party',
|
||||
slug: 'crueltyparty',
|
||||
url: 'https://pornpros.com/site/crueltyparty',
|
||||
url: 'https://pornpros.com/sites/crueltyparty',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Disgraced 18',
|
||||
slug: 'disgraced18',
|
||||
url: 'https://pornpros.com/site/disgraced18',
|
||||
url: 'https://pornpros.com/sites/disgraced18',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Cumshot Surprise',
|
||||
slug: 'cumshotsurprise',
|
||||
url: 'https://pornpros.com/site/cumshotsurprise',
|
||||
url: 'https://pornpros.com/sites/cumshotsurprise',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: '40oz Bounce',
|
||||
slug: 'fortyozbounce',
|
||||
url: 'https://pornpros.com/site/40ozbounce',
|
||||
url: 'https://pornpros.com/sites/40ozbounce',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Jurassic Cock',
|
||||
slug: 'jurassiccock',
|
||||
url: 'https://pornpros.com/site/jurassiccock',
|
||||
url: 'https://pornpros.com/sites/jurassiccock',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Freaks Of Cock',
|
||||
slug: 'freaksofcock',
|
||||
url: 'https://pornpros.com/site/freaksofcock',
|
||||
url: 'https://pornpros.com/sites/freaksofcock',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Euro Humpers',
|
||||
slug: 'eurohumpers',
|
||||
url: 'https://pornpros.com/site/eurohumpers',
|
||||
url: 'https://pornpros.com/sites/eurohumpers',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Freaks Of Boobs',
|
||||
slug: 'freaksofboobs',
|
||||
url: 'https://pornpros.com/site/freaksofboobs',
|
||||
url: 'https://pornpros.com/sites/freaksofboobs',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Cock Competition',
|
||||
slug: 'cockcompetition',
|
||||
url: 'https://pornpros.com/site/cockcompetition',
|
||||
url: 'https://pornpros.com/sites/cockcompetition',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
name: 'Pimp Parade',
|
||||
slug: 'pimpparade',
|
||||
url: 'https://pornpros.com/site/pimpparade',
|
||||
url: 'https://pornpros.com/sites/pimpparade',
|
||||
parent: 'pornpros',
|
||||
},
|
||||
{
|
||||
|
|
@ -9445,8 +9445,7 @@ const sites = [
|
|||
url: 'https://flexiblepositions.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
parent: true,
|
||||
latest: 'https://pornpros.com/site/flexiblepositions',
|
||||
latest: 'https://pornpros.com/sites/flexiblepositions',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -9455,8 +9454,7 @@ const sites = [
|
|||
url: 'https://publicviolations.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
parent: true,
|
||||
latest: 'https://pornpros.com/site/publicviolations',
|
||||
latest: 'https://pornpros.com/sites/publicviolations',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -9464,6 +9462,9 @@ const sites = [
|
|||
slug: 'amateurviolations',
|
||||
url: 'https://amateurviolations.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
latest: 'https://pornpros.com/sites/amateurviolations',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Squirt Disgrace',
|
||||
|
|
@ -9471,8 +9472,7 @@ const sites = [
|
|||
url: 'https://squirtdisgrace.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
parent: true,
|
||||
latest: 'https://pornpros.com/site/squirtdisgrace',
|
||||
latest: 'https://pornpros.com/sites/squirtdisgrace',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -9480,12 +9480,18 @@ const sites = [
|
|||
slug: 'cumdisgrace',
|
||||
url: 'https://cumdisgrace.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
latest: 'https://pornpros.com/sites/cumdisgrace',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Webcam Hackers',
|
||||
slug: 'webcamhackers',
|
||||
url: 'https://webcamhackers.com',
|
||||
parent: 'pornpros',
|
||||
parameters: {
|
||||
latest: 'https://pornpros.com/sites/webcamhackers',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'College Teens',
|
||||
|
|
@ -14608,6 +14614,9 @@ const sites = [
|
|||
url: 'https://facials4k.com',
|
||||
tags: ['fake-cum', 'facial', '4k'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://facials4k.com/sites/facials4k',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Cum 4K',
|
||||
|
|
@ -14615,6 +14624,9 @@ const sites = [
|
|||
url: 'https://cum4k.com',
|
||||
tags: ['fake-cum', 'creampie', '4k'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://cum4k.com/sites/cum4k',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Anal 4K',
|
||||
|
|
@ -14622,6 +14634,9 @@ const sites = [
|
|||
url: 'https://anal4k.com',
|
||||
tags: ['anal', '4k'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://anal4k.com/sites/anal-4k',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Tiny 4K',
|
||||
|
|
@ -14629,6 +14644,9 @@ const sites = [
|
|||
url: 'https://tiny4k.com',
|
||||
tags: ['4k'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://tiny4k.com/sites/tiny4k',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'POVD',
|
||||
|
|
@ -14636,6 +14654,9 @@ const sites = [
|
|||
url: 'https://povd.com',
|
||||
tags: ['pov'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://povd.com/sites/povd',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Lubed',
|
||||
|
|
@ -14643,6 +14664,9 @@ const sites = [
|
|||
url: 'https://lubed.com',
|
||||
tags: ['oil'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://lubed.com/sites/lubed',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Casting Couch X',
|
||||
|
|
@ -14650,6 +14674,9 @@ const sites = [
|
|||
alias: ['castingcouch x', 'castingcouch-x', 'casting couch-x'],
|
||||
url: 'https://castingcouch-x.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://castingcouch-x.com/sites/casting-couch-x',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Passion HD',
|
||||
|
|
@ -14657,18 +14684,27 @@ const sites = [
|
|||
alias: ['phd', 'passion-hd'],
|
||||
url: 'https://passion-hd.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://passion-hd.com/sites/passion-hd',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Nanny Spy',
|
||||
slug: 'nannyspy',
|
||||
url: 'https://nannyspy.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'http://nannyspy.com/sites/nannyspy',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Girl Cum',
|
||||
slug: 'girlcum',
|
||||
url: 'https://girlcum.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://girlcum.com/sites/girlcum',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Pure Mature',
|
||||
|
|
@ -14676,6 +14712,9 @@ const sites = [
|
|||
url: 'https://puremature.com',
|
||||
tags: ['milf'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://puremature.com/sites/pure-mature',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Fantasy HD',
|
||||
|
|
@ -14683,6 +14722,9 @@ const sites = [
|
|||
alias: ['fhd'],
|
||||
url: 'https://fantasyhd.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://fantasyhd.com/sites/fantasy-hd',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Spy Fam',
|
||||
|
|
@ -14690,6 +14732,9 @@ const sites = [
|
|||
url: 'https://spyfam.com',
|
||||
tags: ['family'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://spyfam.com/sites/spyfam',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Holed',
|
||||
|
|
@ -14697,6 +14742,9 @@ const sites = [
|
|||
url: 'https://holed.com',
|
||||
tags: ['anal'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://holed.com/sites/holed',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'BBC Pie',
|
||||
|
|
@ -14704,6 +14752,9 @@ const sites = [
|
|||
url: 'https://bbcpie.com',
|
||||
tags: ['bbc', 'interracial'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://bbcpie.com/sites/bbc-pie',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Wet VR',
|
||||
|
|
@ -14711,6 +14762,9 @@ const sites = [
|
|||
url: 'https://wetvr.com',
|
||||
tags: ['vr'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://wetvr.com/sites/wetvr',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Exotic 4K',
|
||||
|
|
@ -14718,6 +14772,9 @@ const sites = [
|
|||
url: 'https://exotic4k.com',
|
||||
tags: ['4k'],
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://exotic4k.com/sites/exotic4k',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'My Very First Time',
|
||||
|
|
@ -14725,6 +14782,9 @@ const sites = [
|
|||
alias: ['mvft'],
|
||||
url: 'https://myveryfirsttime.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://myveryfirsttime.com/sites/my-very-first-time',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Baeb',
|
||||
|
|
@ -14732,6 +14792,9 @@ const sites = [
|
|||
alias: ['baebz'],
|
||||
url: 'https://baeb.com',
|
||||
parent: 'whalemember',
|
||||
parameters: {
|
||||
latest: 'https://baeb.com/sites/baeb',
|
||||
},
|
||||
},
|
||||
// WICKED
|
||||
{
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ async function init() {
|
|||
|
||||
await http.destroyBypassSessions();
|
||||
await http.destroyBrowserSessions();
|
||||
await unprint.closeAllBrowsers();
|
||||
|
||||
knex.destroy();
|
||||
redis.disconnect();
|
||||
|
|
|
|||
|
|
@ -5,26 +5,38 @@ const unprint = require('unprint');
|
|||
const { stripQuery } = require('../utils/url');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeLatest(scenes, channel) {
|
||||
return scenes.map(({ query, _element }) => {
|
||||
const dateRegex = /\d{4}-\d{2}-\d{2}T/;
|
||||
|
||||
function scrapeLatest(scenes, fullData, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('[href*="/video"]');
|
||||
// release.entryId = unprint.query.attribute(element, null, 'data-vid'); // does not match old videos
|
||||
|
||||
release.title = query.content('.video-thumbnail-footer a[href*="/video"]');
|
||||
release.date = query.date('.actor-list + span', 'MM/DD/YYYY');
|
||||
release.url = query.url('[href*="/video"]', { origin: new URL(channel.url).origin });
|
||||
release.title = query.content('a[href*="/video"] strong');
|
||||
|
||||
release.entryId = release.url
|
||||
? new URL(release.url).pathname.split('/').at(-1)
|
||||
: slugify(release.title);
|
||||
|
||||
release.actors = query.all('.actor-list a').map((actorEl) => ({
|
||||
// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
|
||||
const dataIndex = fullData?.indexOf(release.entryId);
|
||||
const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 35) : [];
|
||||
|
||||
// older scenes don't have date in html
|
||||
const date = data.find((item) => dateRegex.test(item));
|
||||
|
||||
if (date) {
|
||||
release.date = new Date(date);
|
||||
} else {
|
||||
release.date = query.date('a[href*="/video"] + p + p', 'MM/DD/YYYY');
|
||||
}
|
||||
|
||||
release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.poster() || query.img('a img', { attribute: 'data-src' });
|
||||
const poster = query.img('img[alt]');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
|
|
@ -33,99 +45,107 @@ function scrapeLatest(scenes, channel) {
|
|||
];
|
||||
}
|
||||
|
||||
release.photos = query.imgs('img[data-index]', { attribute: 'data-src' }).map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.teaser = query.video('source', { attribute: 'data-src' });
|
||||
// photos and teasers can't be reliably extracted, MP4s include trailers and FULL SCENES
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeLatestPlus(scenes, channel) {
|
||||
return scenes.map(({ query, _element }) => {
|
||||
const release = {};
|
||||
async function passAgeCheck(ctx) {
|
||||
const ageButton = await ctx.getByText('Continue', { exact: true });
|
||||
|
||||
release.title = query.content('.title');
|
||||
release.date = query.date('.video-thumbnail-footer', 'MM/DD/YYYY');
|
||||
|
||||
const slug = slugify(release.title);
|
||||
|
||||
release.url = `${new URL(channel.url).origin}/video/${slug}`;
|
||||
release.entryId = slug;
|
||||
|
||||
release.actors = query.all('.actor-list a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.poster() || query.img('a[data-video-thumbnail-single-image-target] img');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
}
|
||||
|
||||
release.teaser = query.video('source', { attribute: 'data-src' });
|
||||
|
||||
release.channel = slugify(query.attribute('.series-link img', 'alt'), '');
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query }, { url, entity }) {
|
||||
const release = {};
|
||||
|
||||
// release.entryId = query.attribute('div[data-id]', 'data-id');
|
||||
release.entryId = new URL(url).pathname.split('/').at(-1);
|
||||
|
||||
release.title = query.content('.scene-info h1');
|
||||
release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span');
|
||||
|
||||
release.duration = (query.number('//div[contains(@class, \'scene-info\')]//span[contains(text(), \'Duration\')]/following-sibling::span[contains(text(), \'minutes\')]') * 60) || null;
|
||||
|
||||
release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: entity.url }),
|
||||
}));
|
||||
|
||||
release.poster = query.poster('#player-wrapper video');
|
||||
|
||||
release.photos = query.imgs('#trailer_player .hidden > a img').map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.teaser = query.video('#player-wrapper source');
|
||||
release.qualities = query.contents('#trailer_player .resolution').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
if (await ageButton.count() > 0) {
|
||||
await ageButton.click();
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.parameters?.latest || channel.url}?page=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '//*[(starts-with(text(), \'Latest\') and contains(text(), \'Movies\')) or contains(text(), \'Most Recent\')]/following::div[contains(@class, \'video-thumbnail\') and @data-vid]' });
|
||||
// going to e.g. https://holed.com/sites/holed defined by parameter gets rid of 'top rated' section, simplifying query
|
||||
const url = `${channel.parameters?.latest || channel.url.replace('/series', '/sites')}?page=${page}`;
|
||||
|
||||
// site uses Nuxt without SSR, easiest to render in browser
|
||||
const res = await unprint.browserRequest(url, {
|
||||
async control(ctx) {
|
||||
await passAgeCheck(ctx);
|
||||
},
|
||||
});
|
||||
|
||||
if (res.status === 200) {
|
||||
return scrapeLatest(res.context, channel);
|
||||
const scenes = unprint.initAll(res.context.query.all('.card-grid > div'));
|
||||
const data = res.context.query.json('#__NUXT_DATA__');
|
||||
|
||||
return scrapeLatest(scenes, data, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestPlus(channel, page = 1) {
|
||||
const url = `${channel.url}?page=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.video-thumbnail' });
|
||||
function scrapeScene({ query }, { url, entity }) {
|
||||
const release = {};
|
||||
const { query: infoQuery } = unprint.init(query.element('//div[./*/span[contains(text(), \'Featuring\')]]'));
|
||||
|
||||
if (res.status === 200) {
|
||||
return scrapeLatestPlus(res.context, channel);
|
||||
// release.entryId = query.attribute('div[data-id]', 'data-id');
|
||||
release.entryId = new URL(url).pathname.split('/').at(-1);
|
||||
|
||||
release.title = infoQuery.content('h2');
|
||||
|
||||
const description = infoQuery.content('h2 + p + p');
|
||||
|
||||
if (!description.toLowerCase().includes('n/a')) {
|
||||
release.description = description;
|
||||
}
|
||||
|
||||
// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
|
||||
const fullData = query.json('#__NUXT_DATA__');
|
||||
const dataIndex = fullData?.indexOf(release.entryId);
|
||||
const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 50) : [];
|
||||
|
||||
const date = data.find((item) => dateRegex.test(item));
|
||||
|
||||
if (date) {
|
||||
release.date = new Date(date);
|
||||
}
|
||||
|
||||
release.actors = infoQuery.all('a[href*="/models"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: entity.url }),
|
||||
}));
|
||||
|
||||
const poster = query.img('media-poster img');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
}
|
||||
|
||||
release.photos = Array.from(new Set(query.imgs('a img[src*="content/videos"]'))).map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.trailer = query.video('media-player video');
|
||||
release.qualities = query.contents('//table[.//span[contains(text(), \'480p\')]]//tr').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity) {
|
||||
const res = await unprint.browserRequest(url, {
|
||||
async control(ctx) {
|
||||
await passAgeCheck(ctx);
|
||||
|
||||
try {
|
||||
await ctx.locator('media-player video').hover({ trial: true, timeout: 1000 }); // wait for trailer to initialize
|
||||
} catch (__error) {
|
||||
// no trailer, that's fine
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, { url, entity });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
|
@ -133,9 +153,5 @@ async function fetchLatestPlus(channel, page = 1) {
|
|||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
scrapeScene,
|
||||
plus: {
|
||||
fetchLatest: fetchLatestPlus,
|
||||
scrapeScene,
|
||||
},
|
||||
fetchScene,
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue