DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Improved Vixen and XEmpire scrapers. Added media to Blowpass scraper. Improved release fetch code.

Browse Source
This commit is contained in:
ThePendulum
2019-09-26 03:27:01 +02:00
parent 96d8cfbcb6
commit 298eabe56e
11 changed files with 123 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/scrapers/blowpass.js
View File

@@ -21,6 +21,9 @@ function scrape(html, site) {
const date = moment.utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY').toDate();
const actors = $(element).find('.sceneActors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const poster = $(element).find('a.imgLink img.img').attr('data-original');
const trailer = `https://videothumb.gammacdn.com/600x339/${entryId}.mp4`;
const likes = Number($(element).find('.rating .state_1 .value').text());
return {
@@ -29,6 +32,11 @@ function scrape(html, site) {
title,
actors,
date,
poster,
trailer: {
src: trailer,
quality: 339,
},
rating: {
likes,
},
@@ -43,26 +51,26 @@ async function scrapeScene(html, url, site) {
const data = JSON.parse(json).slice(-1)[0];
const sceneElement = $('#wrapper');
const videoScript = $('script:contains("window.ScenePlayerOptions")').html();
const playerObject = videoScript.slice(videoScript.indexOf('{'), videoScript.indexOf('};') + 1);
const playerData = JSON.parse(playerObject);
const workName = data.isPartOf.name.split(' - ');
const shootId = workName.length > 1 ? workName[0] : null;
const shootId = workName.length > 1 ? workName[1] : null;
const entryId = url.split('/').slice(-1)[0];
const title = data.isPartOf ? data.isPartOf.name : data.name;
const { description } = data;
const date = moment.utc(data.isPartOf.datePublished, 'YYYY-MM-DD').toDate();
const title = data.title || $('meta[name="twitter:title"]').attr('content');
const description = data.description || $('meta[name="twitter:description"]').attr('content');
// date in data object is not the release date of the scene, but the date the entry was added
const date = moment.utc($('.updatedDate').first().text(), 'MM-DD-YYYY').toDate();
// const actors = sceneElement.find('.sceneActors a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const actors = data.actor
.sort(({ gender: genderA }, { gender: genderB }) => {
if (genderA === 'female' && genderB === 'male') return -1;
if (genderA === 'male' && genderB === 'female') return 1;
return 0;
})
.map(actor => actor.name);
const actors = data.actor.map(({ name }) => name);
const likes = Number(sceneElement.find('.rating .state_1 .value').text());
const dislikes = Number(sceneElement.find('.rating .state_2 .value').text());
const poster = playerData.picPreview;
const trailer = `${playerData.playerOptions.host}${playerData.url}`;
const duration = moment.duration(data.duration.slice(2)).asSeconds();
const rawTags = data.keywords.split(', ');
@@ -77,6 +85,11 @@ async function scrapeScene(html, url, site) {
actors,
date,
duration,
poster,
trailer: {
src: trailer,
quality: playerData.sizeOnLoad.slice(0, -1),
},
tags,
rating: {
likes,
@@ -87,13 +100,13 @@ async function scrapeScene(html, url, site) {
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.id}/latest/All-Categories/0/All-Pornstars/0/${page}`);
const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.slug}/latest/All-Categories/0/All-Pornstars/0/${page}`);
return scrape(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.id}/upcoming`);
const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.slug}/upcoming`);
return scrape(res.body.toString(), site);
}

14
src/scrapers/vixen.js
View File

@@ -10,8 +10,8 @@ const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const stateObject = $('script:contains("INITIAL_STATE")');
const { videos: scenes } = JSON.parse(stateObject.html().trim().slice(27, -1));
const stateScript = $('script:contains("INITIAL_STATE")').html();
const { videos: scenes } = JSON.parse(stateScript.slice(stateScript.indexOf('{'), stateScript.indexOf('};') + 1));
return scenes.map((scene) => {
const shootId = String(scene.newId);
@@ -59,7 +59,8 @@ async function scrapeScene(html, url, site) {
const shootId = data.page.data[`${pathname}${search}`].data.video;
const scene = data.videos.find(video => video.newId === shootId);
// console.log(scene);
const [poster, ...photos] = scene.rotatingThumbsUrlSizes.map(photo => photo['1040w']);
const trailer = scene.previews.listing.find(preview => preview.height === 353) || null;
const {
title,
@@ -84,6 +85,13 @@ async function scrapeScene(html, url, site) {
date,
duration,
tags,
photos,
poster,
trailer: trailer && {
src: trailer.src,
type: trailer.type,
quality: 353,
},
rating: {
stars,
},

5
src/scrapers/xempire.js
View File

@@ -54,7 +54,9 @@ async function scrapeScene(html, url, site) {
const entryId = new URL(url).pathname.split('/').slice(-1)[0];
const title = $('meta[name="twitter:title"]').attr('content');
const date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate();
const description = data.description || $('meta[name="twitter:description"]').attr('content');
// date in data object is not the release date of the scene, but the date the entry was added
const date = moment.utc($('.updatedDate').first().text(), 'MM-DD-YYYY').toDate();
const actors = data.actor
.sort(({ gender: genderA }, { gender: genderB }) => {
@@ -65,7 +67,6 @@ async function scrapeScene(html, url, site) {
})
.map(actor => actor.name);
const description = data.description || undefined;
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();