Added PornPros satellite sites.

src/scrape-sites.js

@@ -43,7 +43,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
         .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
             && moment(release.date).isAfter(afterDate)); // release is older than specified date limit
 
-    logger.info(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`);
+    logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
 
     if (
         uniqueReleases.length > 0

src/scrapers/pornpros.js

@@ -3,14 +3,18 @@
 const bhttp = require('bhttp');
 const { JSDOM } = require('jsdom');
 const moment = require('moment');
+const ex = require('../utils/ex');
 
 function scrapeLatest(html, site) {
-    const { document } = new JSDOM(html).window;
+    const s = ex(html);
+    const { origin } = new URL(site.url);
 
-    return Array.from(document.querySelectorAll('.site-list .card'), (scene) => {
+    const videos = s.qa('.video-releases-list').slice(-1)[0];
+
+    return Array.from(videos.querySelectorAll('.card'), (scene) => {
         const release = { site };
 
-        release.url = `https://pornpros.com${scene.querySelector(':scope > a').href}`;
+        release.url = `${origin}${scene.querySelector(':scope > a').href}`;
         release.entryId = scene.dataset.videoId;
         release.title = scene.querySelector('.card-title').textContent;
         release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
@@ -26,18 +30,25 @@ function scrapeLatest(html, site) {
     });
 }
 
-function scrapeScene(html, site) {
+function scrapeScene(html, site, url) {
     const { document } = new JSDOM(html).window;
     const release = { site };
 
     const scene = document.querySelector('#t2019-2col');
 
+    release.url = url;
     release.title = scene.querySelector('.t2019-stitle').textContent.trim();
     release.description = scene.querySelector('#t2019-description').textContent.trim();
     release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent);
 
-    const durationEl = scene.querySelector('#t2019-stime');
-    if (durationEl) release.duration = Number(durationEl.textContent.match(/\d+/)[0]) * 60;
+    const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
+
+    if (durationEls.length > 1) {
+        release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
+        release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
+    } else {
+        release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
+    }
 
     release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => `https:${el.src}`);
 
@@ -54,21 +65,21 @@ function scrapeScene(html, site) {
 }
 
 async function fetchLatest(site, page = 1) {
-    const url = `https://pornpros.com/site/${site.slug}?page=${page}`;
+    const url = `${site.url}?page=${page}`;
     const res = await bhttp.get(url);
 
     if (res.statusCode === 200) {
         return scrapeLatest(res.body.toString(), site);
     }
 
-    return null;
+    return [];
 }
 
 async function fetchScene(url, site) {
     const res = await bhttp.get(url);
 
     if (res.statusCode === 200) {
-        return scrapeScene(res.body.toString(), site);
+        return scrapeScene(res.body.toString(), site, url);
     }
 
     return null;

src/sites.js

@@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
 }
 
 async function findSiteByUrl(url) {
-    const { hostname } = new URL(url);
+    const { hostname, origin } = new URL(url);
     const domain = hostname.replace(/www.|tour./, '');
 
     const sites = await knex('sites')
@@ -70,6 +70,7 @@ async function findSiteByUrl(url) {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .where('sites.url', 'like', `${domain}`)
+        .orWhere('sites.url', 'like', `${origin}`)
         .orWhere('sites.url', url);
 
     if (sites.length > 0) {

src/utils/ex.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const { JSDOM } = require('jsdom');
+
+function q(context, selector) {
+    return context.querySelector(selector);
+}
+
+function qa(context, selector) {
+    return Array.from(context.querySelectorAll(selector));
+}
+
+function ex(html) {
+    const { document } = new JSDOM(html).window;
+
+    return {
+        document,
+        q: selector => q(document, selector),
+        qa: selector => qa(document, selector),
+    };
+}
+
+module.exports = ex;