Refactored Whale Member (Porn Pros) scraper.
This commit is contained in:
DebaucheryLibrarian
@@ -229,6 +229,7 @@ async function init() {
|
||||
|
||||
await http.destroyBypassSessions();
|
||||
await http.destroyBrowserSessions();
|
||||
await unprint.closeAllBrowsers();
|
||||
|
||||
knex.destroy();
|
||||
redis.disconnect();
|
||||
|
||||
@@ -5,26 +5,38 @@ const unprint = require('unprint');
|
||||
const { stripQuery } = require('../utils/url');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeLatest(scenes, channel) {
|
||||
return scenes.map(({ query, _element }) => {
|
||||
const dateRegex = /\d{4}-\d{2}-\d{2}T/;
|
||||
|
||||
function scrapeLatest(scenes, fullData, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('[href*="/video"]');
|
||||
// release.entryId = unprint.query.attribute(element, null, 'data-vid'); // does not match old videos
|
||||
|
||||
release.title = query.content('.video-thumbnail-footer a[href*="/video"]');
|
||||
release.date = query.date('.actor-list + span', 'MM/DD/YYYY');
|
||||
release.url = query.url('[href*="/video"]', { origin: new URL(channel.url).origin });
|
||||
release.title = query.content('a[href*="/video"] strong');
|
||||
|
||||
release.entryId = release.url
|
||||
? new URL(release.url).pathname.split('/').at(-1)
|
||||
: slugify(release.title);
|
||||
|
||||
release.actors = query.all('.actor-list a').map((actorEl) => ({
|
||||
// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
|
||||
const dataIndex = fullData?.indexOf(release.entryId);
|
||||
const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 35) : [];
|
||||
|
||||
// older scenes don't have date in html
|
||||
const date = data.find((item) => dateRegex.test(item));
|
||||
|
||||
if (date) {
|
||||
release.date = new Date(date);
|
||||
} else {
|
||||
release.date = query.date('a[href*="/video"] + p + p', 'MM/DD/YYYY');
|
||||
}
|
||||
|
||||
release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.poster() || query.img('a img', { attribute: 'data-src' });
|
||||
const poster = query.img('img[alt]');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
@@ -33,99 +45,107 @@ function scrapeLatest(scenes, channel) {
|
||||
];
|
||||
}
|
||||
|
||||
release.photos = query.imgs('img[data-index]', { attribute: 'data-src' }).map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.teaser = query.video('source', { attribute: 'data-src' });
|
||||
// photos and teasers can't be reliably extracted, MP4s include trailers and FULL SCENES
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeLatestPlus(scenes, channel) {
|
||||
return scenes.map(({ query, _element }) => {
|
||||
const release = {};
|
||||
async function passAgeCheck(ctx) {
|
||||
const ageButton = await ctx.getByText('Continue', { exact: true });
|
||||
|
||||
release.title = query.content('.title');
|
||||
release.date = query.date('.video-thumbnail-footer', 'MM/DD/YYYY');
|
||||
|
||||
const slug = slugify(release.title);
|
||||
|
||||
release.url = `${new URL(channel.url).origin}/video/${slug}`;
|
||||
release.entryId = slug;
|
||||
|
||||
release.actors = query.all('.actor-list a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
const poster = query.poster() || query.img('a[data-video-thumbnail-single-image-target] img');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
}
|
||||
|
||||
release.teaser = query.video('source', { attribute: 'data-src' });
|
||||
|
||||
release.channel = slugify(query.attribute('.series-link img', 'alt'), '');
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query }, { url, entity }) {
|
||||
const release = {};
|
||||
|
||||
// release.entryId = query.attribute('div[data-id]', 'data-id');
|
||||
release.entryId = new URL(url).pathname.split('/').at(-1);
|
||||
|
||||
release.title = query.content('.scene-info h1');
|
||||
release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span');
|
||||
|
||||
release.duration = (query.number('//div[contains(@class, \'scene-info\')]//span[contains(text(), \'Duration\')]/following-sibling::span[contains(text(), \'minutes\')]') * 60) || null;
|
||||
|
||||
release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: entity.url }),
|
||||
}));
|
||||
|
||||
release.poster = query.poster('#player-wrapper video');
|
||||
|
||||
release.photos = query.imgs('#trailer_player .hidden > a img').map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.teaser = query.video('#player-wrapper source');
|
||||
release.qualities = query.contents('#trailer_player .resolution').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
if (await ageButton.count() > 0) {
|
||||
await ageButton.click();
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.parameters?.latest || channel.url}?page=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '//*[(starts-with(text(), \'Latest\') and contains(text(), \'Movies\')) or contains(text(), \'Most Recent\')]/following::div[contains(@class, \'video-thumbnail\') and @data-vid]' });
|
||||
// going to e.g. https://holed.com/sites/holed defined by parameter gets rid of 'top rated' section, simplifying query
|
||||
const url = `${channel.parameters?.latest || channel.url.replace('/series', '/sites')}?page=${page}`;
|
||||
|
||||
// site uses Nuxt without SSR, easiest to render in browser
|
||||
const res = await unprint.browserRequest(url, {
|
||||
async control(ctx) {
|
||||
await passAgeCheck(ctx);
|
||||
},
|
||||
});
|
||||
|
||||
if (res.status === 200) {
|
||||
return scrapeLatest(res.context, channel);
|
||||
const scenes = unprint.initAll(res.context.query.all('.card-grid > div'));
|
||||
const data = res.context.query.json('#__NUXT_DATA__');
|
||||
|
||||
return scrapeLatest(scenes, data, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestPlus(channel, page = 1) {
|
||||
const url = `${channel.url}?page=${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.video-thumbnail' });
|
||||
function scrapeScene({ query }, { url, entity }) {
|
||||
const release = {};
|
||||
const { query: infoQuery } = unprint.init(query.element('//div[./*/span[contains(text(), \'Featuring\')]]'));
|
||||
|
||||
if (res.status === 200) {
|
||||
return scrapeLatestPlus(res.context, channel);
|
||||
// release.entryId = query.attribute('div[data-id]', 'data-id');
|
||||
release.entryId = new URL(url).pathname.split('/').at(-1);
|
||||
|
||||
release.title = infoQuery.content('h2');
|
||||
|
||||
const description = infoQuery.content('h2 + p + p');
|
||||
|
||||
if (!description.toLowerCase().includes('n/a')) {
|
||||
release.description = description;
|
||||
}
|
||||
|
||||
// Nuxt data array does not have a predictable structure, don't rely on it more than necessary
|
||||
const fullData = query.json('#__NUXT_DATA__');
|
||||
const dataIndex = fullData?.indexOf(release.entryId);
|
||||
const data = dataIndex > -1 ? fullData?.slice(dataIndex - 5, dataIndex + 50) : [];
|
||||
|
||||
const date = data.find((item) => dateRegex.test(item));
|
||||
|
||||
if (date) {
|
||||
release.date = new Date(date);
|
||||
}
|
||||
|
||||
release.actors = infoQuery.all('a[href*="/models"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null, { origin: entity.url }),
|
||||
}));
|
||||
|
||||
const poster = query.img('media-poster img');
|
||||
|
||||
if (poster) {
|
||||
release.poster = [
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
}
|
||||
|
||||
release.photos = Array.from(new Set(query.imgs('a img[src*="content/videos"]'))).map((src) => [
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
|
||||
release.trailer = query.video('media-player video');
|
||||
release.qualities = query.contents('//table[.//span[contains(text(), \'480p\')]]//tr').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity) {
|
||||
const res = await unprint.browserRequest(url, {
|
||||
async control(ctx) {
|
||||
await passAgeCheck(ctx);
|
||||
|
||||
try {
|
||||
await ctx.locator('media-player video').hover({ trial: true, timeout: 1000 }); // wait for trailer to initialize
|
||||
} catch (__error) {
|
||||
// no trailer, that's fine
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, { url, entity });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -133,9 +153,5 @@ async function fetchLatestPlus(channel, page = 1) {
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
scrapeScene,
|
||||
plus: {
|
||||
fetchLatest: fetchLatestPlus,
|
||||
scrapeScene,
|
||||
},
|
||||
fetchScene,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user