Added media support to Private scraper.

This commit is contained in:
ThePendulum 2019-11-05 01:41:13 +01:00
parent 3768b7f541
commit f8f6428e36
3 changed files with 34 additions and 2 deletions

View File

@ -145,7 +145,13 @@ function scrollBanner(event) {
function photos() { function photos() {
if (this.release.photos.length) { if (this.release.photos.length) {
return this.release.photos.sort(({ index: indexA }, { index: indexB }) => indexA - indexB); const set = this.release.photos.sort(({ index: indexA }, { index: indexB }) => indexA - indexB);
if (this.release.trailer) {
return set;
}
return [this.release.poster].concat(set);
} }
if (this.release.poster && !this.release.trailer) { if (this.release.poster && !this.release.trailer) {

View File

@ -208,7 +208,7 @@ async function storeReleases(releases = []) {
duration: release.duration, duration: release.duration,
likes: release.rating && release.rating.likes, likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes, dislikes: release.rating && release.rating.dislikes,
rating: release.rating && Math.floor(release.rating.stars), rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: argv.deep, deep: argv.deep,
}; };

View File

@ -8,6 +8,16 @@ const moment = require('moment');
const knex = require('../knex'); const knex = require('../knex');
const { matchTags } = require('../tags'); const { matchTags } = require('../tags');
async function getPhotos(shootId) {
const res = await bhttp.get(`https://www.private.com/gallery.php?type=highres&id=${shootId}`);
const html = res.body.toString();
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = $('a.fakethumb').map((photoIndex, photoElement) => $(photoElement).attr('data-src') || $(photoElement).attr('href')).toArray();
return photos;
}
function scrapeLatest(html, site) { function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.content-wrapper.container .scene').toArray(); const sceneElements = $('.content-wrapper.container .scene').toArray();
@ -19,6 +29,11 @@ function scrapeLatest(html, site) {
const title = sceneLinkElement.text(); const title = sceneLinkElement.text();
const shootId = url.split('/').slice(-1)[0]; const shootId = url.split('/').slice(-1)[0];
const thumbnailElement = $(element).find('img.thumbs_onhover');
const photoCount = Number(thumbnailElement.attr('thumbs_num'));
const poster = thumbnailElement.attr('src');
const photos = Array.from({ length: photoCount }, (val, index) => thumbnailElement.attr(`src${index + 1}`));
const date = moment.utc($(element).find('.scene-date'), 'MM/DD/YYYY').toDate(); const date = moment.utc($(element).find('.scene-date'), 'MM/DD/YYYY').toDate();
const actors = $(element).find('a[data-track="PORNSTAR_LINK"]').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const actors = $(element).find('a[data-track="PORNSTAR_LINK"]').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
@ -30,6 +45,8 @@ function scrapeLatest(html, site) {
title, title,
actors, actors,
date, date,
poster,
photos,
rating: { rating: {
likes, likes,
}, },
@ -51,6 +68,10 @@ async function scrapeScene(html, url, site) {
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g); const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds); const duration = Number(minutes) * 60 + Number(seconds);
const poster = $('meta[property="og:image"]').attr('content');
const trailer = $('meta[property="og:video"]').attr('content');
const photos = await getPhotos(shootId);
const likes = Number($('.content-desc #social-actions #likes').text()); const likes = Number($('.content-desc #social-actions #likes').text());
const siteElement = $('.content-wrapper .logos-sites a'); const siteElement = $('.content-wrapper .logos-sites a');
@ -76,6 +97,11 @@ async function scrapeScene(html, url, site) {
description, description,
duration, duration,
tags, tags,
poster,
photos,
trailer: {
src: trailer,
},
rating: { rating: {
likes, likes,
}, },