Fixed Whale Member (Porn Pros, Holed) scraper.
This commit is contained in:
DebaucheryLibrarian
parent
cc67532fd9
commit
c2afa571bf
|
|
@ -1,145 +1,87 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const unprint = require('unprint');
|
||||||
const moment = require('moment');
|
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const { stripQuery } = require('../utils/url');
|
||||||
|
|
||||||
function scrapeLatest(html, site) {
|
function scrapeLatest(scenes, channel) {
|
||||||
const { document } = new JSDOM(html).window;
|
return scenes.map(({ query, element }) => {
|
||||||
const { origin } = new URL(site.parameters?.latest || site.url);
|
const release = {};
|
||||||
|
|
||||||
const videos = Array.from(document.querySelectorAll('.video-releases-list')).slice(-1)[0];
|
release.url = query.url('[href*="/video"]');
|
||||||
|
release.entryId = unprint.query.attribute(element, null, 'data-vid');
|
||||||
|
|
||||||
return Array.from(videos.querySelectorAll('.card'), (scene) => {
|
release.title = query.content('.video-thumbnail-footer a[href*="/video"]');
|
||||||
const release = { site };
|
release.date = query.date('.actor-list + span', 'MM/DD/YYYY');
|
||||||
|
|
||||||
release.url = `${origin}${scene.querySelector(':scope > a').href}`;
|
release.actors = query.all('.actor-list a').map((actorEl) => ({
|
||||||
release.entryId = scene.dataset.videoId;
|
name: unprint.query.content(actorEl),
|
||||||
release.title = scene.querySelector('.card-title').textContent;
|
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||||
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
|
||||||
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
|
|
||||||
|
|
||||||
// slow CDN?
|
|
||||||
const poster = scene.querySelector('.single-image').dataset.src;
|
|
||||||
const teaserEl = scene.querySelector('source');
|
|
||||||
|
|
||||||
release.poster = {
|
|
||||||
src: /^http/.test(poster) ? poster : `https:${poster}`,
|
|
||||||
referer: site.url,
|
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
};
|
|
||||||
|
|
||||||
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), (el) => ({
|
|
||||||
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
|
|
||||||
referer: site.url,
|
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
if (teaserEl) {
|
const poster = query.poster() || query.img('a img', { attribute: 'data-src' });
|
||||||
release.teaser = {
|
|
||||||
src: teaserEl.dataset.src,
|
if (poster) {
|
||||||
referer: site.url,
|
release.poster = [
|
||||||
attempts: 5,
|
stripQuery(poster),
|
||||||
interval: 5000,
|
poster,
|
||||||
concurrency: 1,
|
];
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
release.photos = query.imgs('img[data-index]', { attribute: 'data-src' }).map((src) => [
|
||||||
|
stripQuery(src),
|
||||||
|
src,
|
||||||
|
]);
|
||||||
|
|
||||||
|
release.teaser = query.video('source', { attribute: 'data-src' });
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeScene(html, site, url) {
|
function scrapeScene({ query }, channel) {
|
||||||
const { document } = new JSDOM(html).window;
|
const release = {};
|
||||||
const release = { site };
|
|
||||||
|
|
||||||
const scene = document.querySelector('#t2019-2col');
|
release.entryId = query.attribute('div[data-id]', 'data-id');
|
||||||
|
|
||||||
release.url = url;
|
release.title = query.content('.scene-info h1');
|
||||||
release.title = scene.querySelector('.t2019-stitle').textContent.trim();
|
release.description = query.content('//div[contains(@class, \'scene-info\')]//i[contains(@class, \'fa-quote\')]/following-sibling::span');
|
||||||
release.description = scene.querySelector('#t2019-description').textContent.trim();
|
|
||||||
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), (el) => el.textContent);
|
|
||||||
|
|
||||||
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
|
release.duration = (query.number('//div[contains(@class, \'scene-info\')]//span[contains(text(), \'Duration\')]/following-sibling::span[contains(text(), \'minutes\')]') * 60) || null;
|
||||||
|
|
||||||
if (durationEls.length > 1) {
|
release.actors = query.all('.scene-info a[href*="/models"]').map((actorEl) => ({
|
||||||
release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
|
name: unprint.query.content(actorEl),
|
||||||
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
|
url: unprint.query.url(actorEl, null, { origin: channel.url }),
|
||||||
} else {
|
|
||||||
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreliable CDN
|
|
||||||
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), (el) => ({
|
|
||||||
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
|
|
||||||
referer: site.url,
|
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const posterEl = scene.querySelector('#no-player-image');
|
release.poster = query.poster('#player-wrapper video');
|
||||||
const videoEl = scene.querySelector('video');
|
|
||||||
const trailerEl = scene.querySelector('#t2019-video source');
|
|
||||||
|
|
||||||
if (posterEl) {
|
release.photos = query.imgs('#trailer_player .hidden > a img').map((src) => [
|
||||||
release.poster = {
|
stripQuery(src),
|
||||||
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
|
src,
|
||||||
referer: site.url,
|
]);
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
};
|
|
||||||
} else if (videoEl) {
|
|
||||||
release.poster = {
|
|
||||||
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
|
|
||||||
referer: site.url,
|
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (trailerEl) {
|
release.teaser = query.video('#player-wrapper source');
|
||||||
release.trailer = {
|
release.qualities = query.contents('#trailer_player .resolution').map((resolution) => Number(resolution.split('x')[1])).filter(Boolean);
|
||||||
src: trailerEl.src,
|
|
||||||
referer: site.url,
|
|
||||||
attempts: 5,
|
|
||||||
interval: 5000,
|
|
||||||
concurrency: 1,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(site, page = 1) {
|
async function fetchLatest(channel, page = 1) {
|
||||||
const url = `${site.parameters?.latest || site.url}?page=${page}`;
|
const url = `${channel.parameters?.latest || channel.url}?page=${page}`;
|
||||||
const res = await http.get(url);
|
const res = await unprint.get(url, { selectAll: '//*[(starts-with(text(), \'Latest\') and contains(text(), \'Movies\')) or contains(text(), \'Most Recent\')]/following::div[contains(@class, \'video-thumbnail\') and @data-vid]' });
|
||||||
|
|
||||||
if (res.statusCode === 200) {
|
if (res.status === 200) {
|
||||||
return scrapeLatest(res.body.toString(), site);
|
return scrapeLatest(res.context, channel);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
return res.status;
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchScene(url, site) {
|
|
||||||
const res = await http.get(url);
|
|
||||||
|
|
||||||
if (res.statusCode === 200) {
|
|
||||||
return scrapeScene(res.body.toString(), site, url);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchScene,
|
scrapeScene: {
|
||||||
|
scraper: scrapeScene,
|
||||||
|
unprint: true,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue