Added Fantasy Massage sites. Improved Private scraper, added movie link.

migrations/20190325001339_releases.js

@@ -105,7 +105,7 @@ exports.up = knex => Promise.resolve()
         table.string('name');
         table.string('url');
         table.text('description');
-        table.string('parameters');
+        table.text('parameters');
 
         table.string('slug', 32)
             .unique();
@@ -140,7 +140,7 @@ exports.up = knex => Promise.resolve()
         table.string('name');
         table.string('url');
         table.text('description');
-        table.string('parameters');
+        table.text('parameters');
 
         table.integer('priority', 3)
             .defaultTo(0);

seeds/01_sites.js

@@ -1457,14 +1457,26 @@ const sites = [
         },
     },
     // FANTASY MASSAGE
+    {
+        slug: 'fantasymassage',
+        name: 'Fantasy Massage',
+        url: 'https://www.fantasymassage.com',
+        network: 'fantasymassage',
+        parameters: {
+            latest: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/AllPornstars/0/updates/',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/Actor/0/upcoming/',
+        },
+    },
     {
         slug: 'allgirlmassage',
         name: 'All Girl Massage',
         url: 'https://www.allgirlmassage.com',
         network: 'fantasymassage',
         parameters: {
-            latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
+            latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/',
-            upcoming: '/en/videos/upcoming',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
         },
     },
     {
@@ -1473,8 +1485,10 @@ const sites = [
         url: 'https://www.nurumassage.com',
         network: 'fantasymassage',
         parameters: {
-            latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
+            latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/',
-            upcoming: '/en/videos/upcoming',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
         },
     },
     {
@@ -1483,9 +1497,10 @@ const sites = [
         url: 'https://www.trickyspa.com',
         network: 'fantasymassage',
         parameters: {
-            latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
+            latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/',
-            upcoming: '/en/videos/upcoming',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/',
-            photos: '/en/photogallery/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
         },
     },
     {
@@ -1494,9 +1509,34 @@ const sites = [
         url: 'https://www.soapymassage.com',
         network: 'fantasymassage',
         parameters: {
-            latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
+            latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/',
-            upcoming: '/en/videos/upcoming',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/',
-            photos: '/en/photogallery/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
+        },
+    },
+    {
+        slug: 'milkingtable',
+        name: 'Milking Table',
+        url: 'https://www.milkingtable.com',
+        network: 'fantasymassage',
+        parameters: {
+            latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
+        },
+    },
+    {
+        slug: 'massageparlor',
+        name: 'Massage Parlor',
+        url: 'https://www.massage-parlor.com',
+        network: 'fantasymassage',
+        parameters: {
+            latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/',
+            upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/',
+            deep: 'network',
+            photos: 'https://www.fantasymassage.com/en/photo/',
         },
     },
     // GIRLSWAY

src/media.js

@@ -38,9 +38,15 @@ function pluckPhotos(photos, specifiedLimit) {
 }
 
 async function getEntropy(buffer) {
+    try {
         const { entropy } = await sharp(buffer).stats();
 
         return entropy;
+    } catch (error) {
+        logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`);
+
+        return 7.5;
+    }
 }
 
 async function createThumbnail(buffer) {
@@ -80,7 +86,7 @@ function curatePhotoEntries(files) {
     }));
 }
 
-async function findDuplicates(photos, identifier, prop = null, label) {
+async function findDuplicates(photos, identifier, prop = null) {
     const duplicates = await knex('media')
         .whereIn(identifier, photos.flat().map((photo) => {
             if (prop) return photo[prop];
@@ -99,14 +105,6 @@ async function findDuplicates(photos, identifier, prop = null, label) {
         return !duplicateLookup.has((prop && source[prop]) || (source.src && source));
     });
 
-    if (duplicates.length > 0) {
-        logger.info(`${duplicates.length} media items already present by ${identifier} for ${label}`);
-    }
-
-    if (originals.length > 0) {
-        logger.info(`Fetching ${originals.length} new media items for ${label}`);
-    }
-
     return [duplicates, originals];
 }
 
@@ -228,6 +226,8 @@ async function storePhotos(photos, {
     const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
     const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
 
+    logger.info(`Fetching ${sourceOriginals.length} new ${role}s, ${sourceDuplicates.length} already present by source for ${label}`);
+
     const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
         concurrency: 10,
     }).filter(photo => photo && photo.entropy > entropyFilter);
@@ -235,6 +235,8 @@ async function storePhotos(photos, {
     const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks
     const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label);
 
+    logger.info(`Saving ${hashOriginals.length} new ${role}s, ${hashDuplicates.length} already present by hash for ${label}`);
+
     const savedPhotos = await savePhotos(hashOriginals, {
         domain,
         role,

src/scrapers/gamma.js

@@ -11,6 +11,16 @@ const logger = require('../logger')(__filename);
 const { ex, get } = require('../utils/q');
 const slugify = require('../utils/slugify');
 
+function getAlbumUrl(albumPath, site) {
+    if (site.parameters?.photos) {
+        return /^http/.test(site.parameters.photos)
+            ? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
+            : `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
+    }
+
+    return `${site.url}${albumPath}`;
+}
+
 async function fetchPhotos(url) {
     const res = await bhttp.get(url);
 
@@ -25,7 +35,7 @@ function scrapePhotos(html) {
 
         if (/\/join|\/createaccount/.test(url)) {
             // URL links to join page instead of full photo, extract thumbnail
-            // /createaccount is used by e.g. Tricky Spa
+            // /createaccount is used by e.g. Tricky Spa native site
             const src = $(linkEl).find('img').attr('src');
 
             if (src.match('previews/')) {
@@ -47,9 +57,9 @@ function scrapePhotos(html) {
 }
 
 async function getPhotos(albumPath, site) {
-    const albumUrl = site.parameters?.photos
+    const albumUrl = getAlbumUrl(albumPath, site);
-        ? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
+
-        : `${site.url}${albumPath}`;
+    console.log(albumUrl);
 
     try {
         const html = await fetchPhotos(albumUrl);
@@ -397,28 +407,63 @@ async function fetchApiUpcoming(site) {
     return fetchApiLatest(site, 1, true);
 }
 
+function getLatestUrl(site, page) {
+    if (site.parameters?.latest) {
+        return /^http/.test(site.parameters.latest)
+            ? `${site.parameters.latest}${page}`
+            : `${site.url}${site.parameters.latest}${page}`;
+    }
+
+    return `${site.url}/en/videos/AllCategories/0/${page}`;
+}
+
+function getUpcomingUrl(site) {
+    if (site.parameters?.upcoming) {
+        return /^http/.test(site.parameters.upcoming)
+            ? `${site.parameters.upcoming}`
+            : `${site.url}${site.parameters.upcoming}`;
+    }
+
+    return `${site.url}/en/videos/AllCategories/0/1/upcoming`;
+}
+
 async function fetchLatest(site, page = 1) {
-    const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`;
+    const url = getLatestUrl(site, page);
     const res = await bhttp.get(url);
 
     return scrapeAll(res.body.toString(), site);
 }
 
 async function fetchUpcoming(site) {
-    const url = `${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`;
+    const url = getUpcomingUrl(site);
     const res = await bhttp.get(url);
 
     return scrapeAll(res.body.toString(), site);
 }
 
+function getDeepUrl(url, site) {
+    const { pathname } = new URL(url);
+
+    if (site.parameters?.deep === 'network') {
+        return `${site.network.url}${pathname}`;
+    }
+
+    if (site.parameters?.deep) {
+        return `${site.parameters.deep}${pathname}`;
+    }
+
+    return url;
+}
+
 async function fetchScene(url, site, release) {
     if (site.parameters?.deep === false) {
         return release;
     }
 
-    const res = await bhttp.get(url);
+    const deepUrl = getDeepUrl(url, site);
+    const res = await bhttp.get(deepUrl);
 
-    return scrapeScene(res.body.toString(), url, site);
+    return scrapeScene(res.body.toString(), url, site, deepUrl);
 }
 
 async function fetchActorScenes(actorName, apiUrl, siteSlug) {

src/scrapers/private.js

@@ -5,9 +5,6 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const fetchSites = require('../sites');
-const { matchTags } = require('../tags');
-
 async function getPhotos(entryId, site) {
     const { hostname } = new URL(site.url);
 
@@ -20,27 +17,6 @@ async function getPhotos(entryId, site) {
     return photos;
 }
 
-async function getChannelSite($, site) {
-    const { hostname } = new URL(site.url);
-
-    if (site.isFallback && hostname.match('private.com')) {
-        const siteElement = $('.content-wrapper .logos-sites a');
-        const siteUrl = siteElement.attr('href').slice(0, -1);
-        const siteName = siteElement.text();
-        const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
-
-        const channelSite = await fetchSites({
-            slug: siteSlug,
-            name: siteName,
-            url: siteUrl,
-        });
-
-        return channelSite;
-    }
-
-    return site;
-}
-
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
     const sceneElements = $('.content-wrapper .scene').toArray();
@@ -82,55 +58,40 @@ function scrapeLatest(html, site) {
 
 async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const release = {};
 
-    const entryId = url.split('/').slice(-1)[0];
+    [release.entryId] = url.split('/').slice(-1);
-    const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
+    release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
+    release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
 
-    const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
+    release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
-    const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
+    release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
 
-    const description = $('.video-wrapper meta[itemprop="description"]').attr('content');
+    const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
-    const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
+
-    const duration = Number(minutes) * 60 + Number(seconds);
+    if (timestamp) {
+        const [minutes, seconds] = timestamp.match(/\d+/g);
+        release.duration = Number(minutes) * 60 + Number(seconds);
+    }
+
+    release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
+    release.likes = Number($('.content-desc #social-actions #likes').text());
 
     const posterScript = $('script:contains(poster)').html();
     const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
-    const poster = $('meta[property="og:image"]').attr('content') || posterLink;
+    release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
 
-    const trailerElementSrc = $('#videojs-trailer source').attr('src');
+    release.trailer = {
-    const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc;
+        src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
-
-    const likes = Number($('.content-desc #social-actions #likes').text());
-
-    const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-
-    const [tags, photos, channelSite] = await Promise.all([
-        matchTags(rawTags),
-        getPhotos(entryId, site),
-        getChannelSite($, site),
-    ]);
-
-    const scene = {
-        url,
-        entryId,
-        title,
-        date,
-        actors,
-        description,
-        duration,
-        tags,
-        poster,
-        photos,
-        trailer: {
-            src: trailer,
-        },
-        rating: {
-            likes,
-        },
-        site: channelSite || site,
     };
 
-    return scene;
+    release.photos = await getPhotos(release.entryId, site);
+    release.movie = $('a[data-track="FULL MOVIE"]').attr('href');
+
+    const siteElement = $('.content-wrapper .logos-sites a');
+    if (siteElement) release.channel = siteElement.text().replace(/\s+/g, '').toLowerCase();
+
+    return release;
 }
 
 async function fetchLatest(site, page = 1) {