Added PornPros satellite sites.

seeds/01_sites.js

@@ -2164,7 +2164,7 @@ function getSites(networksMap) {
         },
         {
             name: 'Disgraced 18',
-            slug: 'disgracedeighteen',
+            slug: 'disgraced18',
             url: 'https://pornpros.com/site/disgraced18',
             network_id: networksMap.pornpros,
         },
@@ -2216,6 +2216,84 @@ function getSites(networksMap) {
             url: 'https://pornpros.com/site/pimpparade',
             network_id: networksMap.pornpros,
         },
+        {
+            name: 'Cum 4K',
+            slug: 'cum4k',
+            url: 'https://cum4k.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Tiny 4K',
+            slug: 'tiny4k',
+            url: 'https://tiny4k.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'POVD',
+            slug: 'povd',
+            url: 'https://povd.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Lubed',
+            slug: 'lubed',
+            url: 'https://lubed.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Casting Couch X',
+            slug: 'castingcouchx',
+            url: 'https://castingcouch-x.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Passion HD',
+            slug: 'passionhd',
+            url: 'https://passion-hd.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Nanny Spy',
+            slug: 'nannyspy',
+            url: 'https://nannyspy.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Girl Cum',
+            slug: 'girlcum',
+            url: 'https://girlcum.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Pure Mature',
+            slug: 'puremature',
+            url: 'https://puremature.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Fantasy HD',
+            slug: 'fantasyhd',
+            url: 'https://fantasyhd.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Spy Fam',
+            slug: 'spyfam',
+            url: 'https://spyfam.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Exotic 4K',
+            slug: 'exotic4k',
+            url: 'https://exotic4k.com',
+            network_id: networksMap.pornpros,
+        },
+        {
+            name: 'Baeb',
+            slug: 'baeb',
+            url: 'https://baeb.com',
+            network_id: networksMap.pornpros,
+        },
         // PRIVATE
         {
             slug: 'analintroductions',

seeds/03_tags.js

@@ -456,6 +456,12 @@ function getTags(groupsMap) {
             alias_for: null,
             group_id: groupsMap.clothing,
         },
+        {
+            name: 'fake cum',
+            slug: 'fake-cum',
+            alias_for: null,
+            group_id: groupsMap.finish,
+        },
         {
             name: 'trainbang',
             slug: 'trainbang',
@@ -611,6 +617,11 @@ function getTags(groupsMap) {
             slug: 'nipple-clamps',
             alias_for: null,
         },
+        {
+            name: 'oil',
+            slug: 'oil',
+            alias_for: null,
+        },
         {
             name: 'oral creampie',
             slug: 'oral-creampie',
@@ -1581,16 +1592,23 @@ function getSiteTags() {
         teenallanal: ['anal', 'mff'],
         boundgods: ['gay'],
         buttmachineboys: ['gay'],
+        cum4k: ['fake-cum', 'creampie', '4k'],
         deepthroatlove: ['blowjob', 'deepthroat'],
         divinebitches: ['femdom'],
         dpparodies: ['parody'],
         eighteenyearsold: ['teen'],
+        exotic4k: ['4k'],
+        lubed: ['oil'],
         familystrokes: ['family'],
         massagecreep: ['massage'],
         menonedge: ['gay'],
+        povd: ['pov'],
+        puremature: ['milf'],
+        spyfam: ['family'],
         submissived: ['bdsm'],
         swallowed: ['blowjob', 'deepthroat', 'facefucking'],
         teenbff: ['mff'],
+        tiny4k: ['4k'],
         trueanal: ['anal'],
         tspussyhunters: ['transsexual'],
     };

src/scrape-sites.js

@@ -43,7 +43,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
         .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
             && moment(release.date).isAfter(afterDate)); // release is older than specified date limit
 
-    logger.info(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`);
+    logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
 
     if (
         uniqueReleases.length > 0

src/scrapers/pornpros.js

@@ -3,14 +3,18 @@
 const bhttp = require('bhttp');
 const { JSDOM } = require('jsdom');
 const moment = require('moment');
+const ex = require('../utils/ex');
 
 function scrapeLatest(html, site) {
-    const { document } = new JSDOM(html).window;
+    const s = ex(html);
+    const { origin } = new URL(site.url);
 
-    return Array.from(document.querySelectorAll('.site-list .card'), (scene) => {
+    const videos = s.qa('.video-releases-list').slice(-1)[0];
+
+    return Array.from(videos.querySelectorAll('.card'), (scene) => {
         const release = { site };
 
-        release.url = `https://pornpros.com${scene.querySelector(':scope > a').href}`;
+        release.url = `${origin}${scene.querySelector(':scope > a').href}`;
         release.entryId = scene.dataset.videoId;
         release.title = scene.querySelector('.card-title').textContent;
         release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
@@ -26,18 +30,25 @@ function scrapeLatest(html, site) {
     });
 }
 
-function scrapeScene(html, site) {
+function scrapeScene(html, site, url) {
     const { document } = new JSDOM(html).window;
     const release = { site };
 
     const scene = document.querySelector('#t2019-2col');
 
+    release.url = url;
     release.title = scene.querySelector('.t2019-stitle').textContent.trim();
     release.description = scene.querySelector('#t2019-description').textContent.trim();
     release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent);
 
-    const durationEl = scene.querySelector('#t2019-stime');
+    const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
-    if (durationEl) release.duration = Number(durationEl.textContent.match(/\d+/)[0]) * 60;
+
+    if (durationEls.length > 1) {
+        release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
+        release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
+    } else {
+        release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
+    }
 
     release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => `https:${el.src}`);
 
@@ -54,21 +65,21 @@ function scrapeScene(html, site) {
 }
 
 async function fetchLatest(site, page = 1) {
-    const url = `https://pornpros.com/site/${site.slug}?page=${page}`;
+    const url = `${site.url}?page=${page}`;
     const res = await bhttp.get(url);
 
     if (res.statusCode === 200) {
         return scrapeLatest(res.body.toString(), site);
     }
 
-    return null;
+    return [];
 }
 
 async function fetchScene(url, site) {
     const res = await bhttp.get(url);
 
     if (res.statusCode === 200) {
-        return scrapeScene(res.body.toString(), site);
+        return scrapeScene(res.body.toString(), site, url);
     }
 
     return null;

src/sites.js

@@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
 }
 
 async function findSiteByUrl(url) {
-    const { hostname } = new URL(url);
+    const { hostname, origin } = new URL(url);
     const domain = hostname.replace(/www.|tour./, '');
 
     const sites = await knex('sites')
@@ -70,6 +70,7 @@ async function findSiteByUrl(url) {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .where('sites.url', 'like', `${domain}`)
+        .orWhere('sites.url', 'like', `${origin}`)
         .orWhere('sites.url', url);
 
     if (sites.length > 0) {

src/utils/ex.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const { JSDOM } = require('jsdom');
+
+function q(context, selector) {
+    return context.querySelector(selector);
+}
+
+function qa(context, selector) {
+    return Array.from(context.querySelectorAll(selector));
+}
+
+function ex(html) {
+    const { document } = new JSDOM(html).window;
+
+    return {
+        document,
+        q: selector => q(document, selector),
+        qa: selector => qa(document, selector),
+    };
+}
+
+module.exports = ex;