Added filter bar to all pages. Added 'upcoming' marker. Improved date tidbit.

src/fetch-releases.js

@@ -103,7 +103,7 @@ async function storeRelease(release) {
         likes: release.rating && release.rating.likes,
         dislikes: release.rating && release.rating.dislikes,
         rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
-        deep: argv.deep,
+        deep: Boolean(argv.deep && release.url && !release.upcoming),
     };
 
     const releaseEntries = await knex('releases')
@@ -201,13 +201,15 @@ async function fetchReleases() {
 
                 const [newReleases, upcomingReleases] = await Promise.all([
                     fetchNewReleases(scraper, site, afterDate),
-                    scraper.fetchUpcoming ? await scraper.fetchUpcoming(site) : [],
+                    scraper.fetchUpcoming ? scraper.fetchUpcoming(site) : [],
                 ]);
 
                 console.log(`${site.name}: Found ${newReleases.length} recent releases, ${upcomingReleases.length} upcoming releases`);
 
+                const markedUpcomingReleases = upcomingReleases.map(release => ({ ...release, upcoming: true }));
+
                 const finalReleases = argv.deep
-                    ? await Promise.map(newReleases, async (release) => {
+                    ? await Promise.map([...newReleases, ...markedUpcomingReleases], async (release) => {
                         if (release.url) {
                             const scene = await fetchScene(release.url, release);
 

src/scrapers/julesjordan.js

@@ -7,6 +7,8 @@ const moment = require('moment');
 
 const { matchTags } = require('../tags');
 
+const pluckPhotos = require('../utils/pluck-photos');
+
 async function fetchPhotos(url) {
     const res = await bhttp.get(url);
 
@@ -23,7 +25,7 @@ function scrapePhotos(html) {
     return photos;
 }
 
-async function getPhotos(entryId, page = 1) {
+async function getPhotos(entryId, site, page = 1) {
     const albumUrl = `https://www.julesjordan.com/trial/gallery.php?id=${entryId}&type=highres&page=${page}`;
 
     const html = await fetchPhotos(albumUrl);
@@ -41,7 +43,14 @@ async function getPhotos(entryId, page = 1) {
         concurrency: 2,
     });
 
-    return photos.concat(otherPhotos.flat());
+    const allPhotos = photos.concat(otherPhotos.flat());
+
+    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
+    const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
+
+    const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
+
+    return pluckedPhotos;
 }
 
 function scrapeLatest(html, site) {
@@ -51,7 +60,7 @@ function scrapeLatest(html, site) {
     return scenesElements.map((element) => {
         const photoElement = $(element).find('a img.thumbs');
         const photoCount = Number(photoElement.attr('cnt'));
-        const photos = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
+        const [poster, ...photos] = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
 
         const sceneLinkElement = $(element).children('a').eq(1);
         const url = sceneLinkElement.attr('href');
@@ -73,8 +82,9 @@ function scrapeLatest(html, site) {
             title,
             actors,
             date,
-            site,
+            poster,
             photos,
+            site,
         };
     });
 }
@@ -84,10 +94,6 @@ function scrapeUpcoming(html, site) {
     const scenesElements = $('#coming_soon_carousel').find('.table').toArray();
 
     return scenesElements.map((element) => {
-        const photoElement = $(element).find('a img.thumbs');
-        const photoCount = Number(photoElement.attr('cnt'));
-        const photos = Array.from({ length: photoCount }, (value, index) => photoElement.attr(`src${index}_1x`)).filter(photoUrl => photoUrl !== undefined);
-
         const entryId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0];
 
         const details = $(element).find('.update_details_comingsoon')
@@ -109,13 +115,23 @@ function scrapeUpcoming(html, site) {
             .utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
             .toDate();
 
+        const photoElement = $(element).find('a img.thumbs');
+        const poster = photoElement.attr('src');
+
+        const videoClass = $(element).find('.update_thumbnail div').attr('class');
+        const videoScript = $(element).find(`script:contains(${videoClass})`).html();
+        const trailer = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
+
         return {
             url: null,
             entryId,
             title,
             date,
             actors,
-            photos,
+            poster,
+            trailer: {
+                src: trailer,
+            },
             rating: null,
             site,
         };
@@ -148,7 +164,7 @@ async function scrapeScene(html, url, site) {
     const trailerLine = infoLines.find(line => line.match('movie["Trailer_720"]'));
     const trailer = trailerLine.slice(trailerLine.indexOf('path:"') + 6, trailerLine.indexOf('",movie'));
 
-    const photos = await getPhotos(entryId);
+    const photos = await getPhotos(entryId, site);
 
     const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
     const tags = await matchTags(rawTags);

src/utils/pluck-photos.js

@@ -2,7 +2,7 @@
 
 // pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
 function pluckPhotos(photoTotal, photoLimit) {
-    return [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.floor((index + 1) * (photoTotal / (photoLimit - 1)))));
+    return [1].concat(Array.from({ length: photoLimit - 1 }, (value, index) => Math.round((index + 1) * (photoTotal / (photoLimit - 1)))));
 }
 
 module.exports = pluckPhotos;

src/web/releases.js

@@ -29,7 +29,7 @@ async function fetchActorReleasesApi(req, res) {
     const releases = await fetchActorReleases({
         id: actorId,
         slug: actorSlug,
-    });
+    }, req.query);
 
     res.send(releases);
 }
@@ -53,7 +53,7 @@ async function fetchSiteReleasesApi(req, res) {
     const releases = await fetchSiteReleases({
         id: siteId,
         slug: siteSlug,
-    });
+    }, req.query);
 
     res.send(releases);
 }
@@ -65,7 +65,7 @@ async function fetchTagReleasesApi(req, res) {
     const releases = await fetchTagReleases({
         id: tagId,
         slug: tagSlug,
-    });
+    }, req.query);
 
     res.send(releases);
 }