DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added filter bar to all pages. Added 'upcoming' marker. Improved date tidbit.

Browse Source
This commit is contained in:
ThePendulum
2019-11-15 05:10:59 +01:00
parent a612045ee0
commit 5620dfcb65
18 changed files with 180 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/scrapers/julesjordan.js
View File

@@ -7,6 +7,8 @@ const moment = require('moment');
const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
async function fetchPhotos(url) {
const res = await bhttp.get(url);
@@ -23,7 +25,7 @@ function scrapePhotos(html) {
return photos;
}
async function getPhotos(entryId, page = 1) {
async function getPhotos(entryId, site, page = 1) {
const albumUrl = `https://www.julesjordan.com/trial/gallery.php?id=${entryId}&type=highres&page=${page}`;
const html = await fetchPhotos(albumUrl);
@@ -41,7 +43,14 @@ async function getPhotos(entryId, page = 1) {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
const allPhotos = photos.concat(otherPhotos.flat());
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
return pluckedPhotos;
}
function scrapeLatest(html, site) {
@@ -51,7 +60,7 @@ function scrapeLatest(html, site) {
return scenesElements.map((element) => {
const photoElement = $(element).find('a img.thumbs');
const photoCount = Number(photoElement.attr('cnt'));
const photos = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
const [poster, ...photos] = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
const sceneLinkElement = $(element).children('a').eq(1);
const url = sceneLinkElement.attr('href');
@@ -73,8 +82,9 @@ function scrapeLatest(html, site) {
title,
actors,
date,
site,
poster,
photos,
site,
};
});
}
@@ -84,10 +94,6 @@ function scrapeUpcoming(html, site) {
const scenesElements = $('#coming_soon_carousel').find('.table').toArray();
return scenesElements.map((element) => {
const photoElement = $(element).find('a img.thumbs');
const photoCount = Number(photoElement.attr('cnt'));
const photos = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
const entryId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0];
const details = $(element).find('.update_details_comingsoon')
@@ -109,13 +115,23 @@ function scrapeUpcoming(html, site) {
.utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
.toDate();
const photoElement = $(element).find('a img.thumbs');
const poster = photoElement.attr('src');
const videoClass = $(element).find('.update_thumbnail div').attr('class');
const videoScript = $(element).find(`script:contains(${videoClass})`).html();
const trailer = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
return {
url: null,
entryId,
title,
date,
actors,
photos,
poster,
trailer: {
src: trailer,
},
rating: null,
site,
};
@@ -148,7 +164,7 @@ async function scrapeScene(html, url, site) {
const trailerLine = infoLines.find(line => line.match('movie["Trailer_720"]'));
const trailer = trailerLine.slice(trailerLine.indexOf('path:"') + 6, trailerLine.indexOf('",movie'));
const photos = await getPhotos(entryId);
const photos = await getPhotos(entryId, site);
const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = await matchTags(rawTags);