Added Team Skeet scraper.

seeds/00_networks.js

@@ -120,6 +120,12 @@ const networks = [
         url: 'https://www.realitykings.com',
         description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
     },
+    {
+        slug: 'teamskeet',
+        name: 'Team Skeet',
+        url: 'https://www.teamskeet.com',
+        description: 'Welcome to teamskeet.com, the largest collection of exclusive teen porn sites and videos on the web. Check out our TeamSkeet porn sites now.',
+    },
     {
         slug: 'vixen',
         name: 'Vixen',

seeds/01_sites.js

@@ -2480,6 +2480,230 @@ function getSites(networksMap) {
             slug: 'monstercurves',
             network_id: networksMap.realitykings,
         },
+        {
+            slug: 'exxxtrasmall',
+            name: 'Exxxtra Small',
+            description: '',
+            url: 'https://www.exxxtrasmall.com',
+            parameters: JSON.stringify({ id: 'exs' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teenpies',
+            name: 'Teen Pies',
+            description: '',
+            url: 'https://www.teenpies.com',
+            parameters: JSON.stringify({ id: 'tp' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'innocenthigh',
+            name: 'Innocent High',
+            description: '',
+            url: 'https://www.innocenthigh.com',
+            parameters: JSON.stringify({ id: 'ih' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teencurves',
+            name: 'Teen Curves',
+            description: '',
+            url: 'https://www.teencurves.com',
+            parameters: JSON.stringify({ id: 'tc' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'cfnmteens',
+            name: 'CFNM Teens',
+            description: '',
+            url: 'https://www.cfnmteens.com',
+            parameters: JSON.stringify({ id: 'cfnm' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teensloveanal',
+            name: 'Teens Love Anal',
+            description: '',
+            url: 'https://www.teensloveanal.com',
+            parameters: JSON.stringify({ id: 'tla' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'mybabysittersclub',
+            name: 'My Babysitters Club',
+            description: '',
+            url: 'https://www.mybabysittersclub.com',
+            parameters: JSON.stringify({ id: 'bsc' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'shesnew',
+            name: 'She\'s New',
+            description: '',
+            url: 'https://www.shesnow.com',
+            parameters: JSON.stringify({ id: 'bsc' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teensdoporn',
+            name: 'Teens Do Porn',
+            description: '',
+            url: 'https://www.teensdoporn.com',
+            parameters: JSON.stringify({ id: 'tdp' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'povlife',
+            name: 'POV Life',
+            description: '',
+            url: 'https://www.povlife.com',
+            parameters: JSON.stringify({ id: 'pov' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'therealworkout',
+            name: 'The Real Workout',
+            description: '',
+            url: 'https://www.therealworkout.com',
+            parameters: JSON.stringify({ id: 'trw' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'thisgirlsucks',
+            name: 'This Girl Sucks',
+            description: '',
+            url: 'https://www.thisgirlsucks.com',
+            parameters: JSON.stringify({ id: 'tgs' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teenslovemoney',
+            name: 'Teens Love Money',
+            description: '',
+            url: 'https://www.teenslovemoney.com',
+            parameters: JSON.stringify({ id: 'tlm' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'oyeloca',
+            name: 'Oye Loca',
+            description: '',
+            url: 'https://www.oyeloca.com',
+            parameters: JSON.stringify({ id: 'ol' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'tittyattack',
+            name: 'Titty Attack',
+            description: '',
+            url: 'https://www.tittyattack.com',
+            parameters: JSON.stringify({ id: 'ta' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teenyblack',
+            name: 'Teeny Black',
+            description: '',
+            url: 'https://www.teenyblack.com',
+            parameters: JSON.stringify({ id: 'tb' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'lusthd',
+            name: 'Lust HD',
+            description: '',
+            url: 'https://www.lusthd.com',
+            parameters: JSON.stringify({ id: 'lhd' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'rubateen',
+            name: 'Rub A Teen',
+            description: '',
+            url: 'https://www.rubateen.com',
+            parameters: JSON.stringify({ id: 'rat' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'herfreshmanyear',
+            name: 'Her Freshman Year',
+            description: '',
+            url: 'https://www.exxxtrasmall.com',
+            parameters: JSON.stringify({ id: 'hfy' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'selfdesire',
+            name: 'Self Desire',
+            description: '',
+            url: 'https://www.selfdesire.com',
+            parameters: JSON.stringify({ id: 'sd' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'solointerviews',
+            name: 'Solo Interviews',
+            description: '',
+            url: 'https://www.solointerviews.com',
+            parameters: JSON.stringify({ id: 'si' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teamskeetextras',
+            name: 'Team Skeet Extras',
+            description: '',
+            url: 'https://www.teamskeetextras.com',
+            parameters: JSON.stringify({ id: 'tse' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'dyked',
+            name: 'Dyked',
+            description: '',
+            url: 'https://www.dyked.com',
+            parameters: JSON.stringify({ id: 'dyk' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'badmilfs',
+            name: 'Bad MILFs',
+            description: '',
+            url: 'https://www.badmilfs.com',
+            parameters: JSON.stringify({ id: 'bad' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'gingerpatch',
+            name: 'Ginger Patch',
+            description: '',
+            url: 'https://www.gingerpatch.com',
+            parameters: JSON.stringify({ id: 'gp' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'bracefaced',
+            name: 'Brace Faced',
+            description: '',
+            url: 'https://www.bracefaced.com',
+            parameters: JSON.stringify({ id: 'bfd' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'teenjoi',
+            name: 'Teen JOI',
+            description: '',
+            url: 'https://www.teenjoi.com',
+            parameters: JSON.stringify({ id: 'joi' }),
+            network_id: networksMap.teamskeet,
+        },
+        {
+            slug: 'stepsiblings',
+            name: 'Step Siblings',
+            description: '',
+            url: 'https://www.stepsiblings.com',
+            parameters: JSON.stringify({ id: 'sss' }),
+            network_id: networksMap.teamskeet,
+        },
         // VIXEN
         {
             slug: 'vixen',

src/scrapers/jayrock.js

@@ -4,6 +4,7 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 
+const logger = require('../logger');
 const slugify = require('../utils/slugify');
 
 async function fetchToken(site) {
@@ -30,12 +31,17 @@ async function fetchActors(entryId, site, { token, time }) {
 
 async function fetchTrailerLocation(entryId, site) {
     const url = `${site.url}/api/download/${entryId}/hd1080/stream`;
-    const res = await bhttp.get(url, {
-        followRedirects: false,
-    });
 
-    if (res.statusCode === 302) {
+    try {
-        return res.headers.location;
+        const res = await bhttp.get(url, {
+            followRedirects: false,
+        });
+
+        if (res.statusCode === 302) {
+            return res.headers.location;
+        }
+    } catch (error) {
+        logger.warn(`${site.name}: Unable to fetch trailer at '${url}': ${error.message}`);
     }
 
     return null;
@@ -72,11 +78,11 @@ function scrapeLatest(scenes, site, tokens) {
     return Promise.map(scenes, async scene => scrapeScene(scene, site, tokens), { concurrency: 10 });
 }
 
-async function fetchLatest(site) {
+async function fetchLatest(site, page = 1) {
     const { time, token } = await fetchToken(site);
 
     // transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
-    const url = `${site.url}/sapi/${token}/${time}/content.load?limit=20&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
+    const url = `${site.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
     const res = await bhttp.get(url);
 
     if (res.statusCode === 200 && res.body.status) {

src/scrapers/scrapers.js

@@ -15,6 +15,7 @@ const pervcity = require('./pervcity');
 const privateNetwork = require('./private'); // reserved keyword
 const naughtyamerica = require('./naughtyamerica');
 const realitykings = require('./realitykings');
+const teamskeet = require('./teamskeet');
 const vixen = require('./vixen');
 
 // releases and profiles
@@ -52,6 +53,7 @@ module.exports = {
         private: privateNetwork,
         naughtyamerica,
         realitykings,
+        teamskeet,
         vixen,
         xempire,
     },

src/scrapers/teamskeet.js

@@ -0,0 +1,105 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const { JSDOM } = require('jsdom');
+const moment = require('moment');
+
+function extractTitle(pathname) {
+    return pathname
+        .split('/')
+        .slice(-2)[0]
+        .split('_')
+        .map(seg => `${seg.charAt(0).toUpperCase()}${seg.slice(1)}`)
+        .join(' ');
+}
+
+function extractActors(str) {
+    return str
+        .split(/,|\band/)
+        .filter(actor => !/\.{3}/.test(actor))
+        .map(actor => actor.trim())
+        .filter(actor => actor.length > 0);
+}
+
+function scrapeLatest(html, site) {
+    const { document } = new JSDOM(html).window;
+
+    const scenes = Array.from(document.querySelectorAll('#updatesList li.grey, #updatesList li.white'));
+
+    return scenes.map((scene) => {
+        const release = { site };
+
+        const link = scene.querySelector('.info a');
+        const poster = scene.querySelector('img');
+        const { pathname } = new URL(link);
+
+        [release.entryId] = poster.id.match(/\d+/);
+
+        release.url = `https://www.teamskeet.com${pathname}`;
+        release.title = extractTitle(pathname);
+
+        release.date = moment.utc(scene.querySelector('strong').textContent, 'MM/DD/YYYY').toDate();
+
+        const photos = Array.from({ length: 5 }, (_value, index) => poster.dataset.original.replace(/\d+.jpg/, `${String(index + 1).padStart(2, '0')}.jpg`));
+        [release.poster] = photos;
+        release.photos = photos.slice(1);
+
+        const actors = scene.querySelector('div span[rel="test"]').textContent;
+        release.actors = extractActors(actors);
+
+        return release;
+    });
+}
+
+function scrapeScene(html, site) {
+    const { document } = new JSDOM(html).window;
+    const release = { site };
+
+    release.entryId = document.querySelector('#story-and-tags .scene_rater').attributes.rel.value;
+    release.description = document.querySelector('#story-and-tags td:nth-child(2) div').textContent;
+    const [actors, title, channel] = document.querySelector('title').textContent.split('|').map(item => item.trim());
+
+    release.title = title;
+    release.actors = extractActors(actors);
+    release.channel = channel.toLowerCase();
+    release.tags = Array.from(document.querySelectorAll('#story-and-tags tr:nth-child(2) a'), el => el.rel);
+
+    const date = document.querySelector('h3 ~ div:nth-child(4), h3 ~ div div.gray:not(.scene_rater)').textContent.split(':')[1].trim();
+    release.date = moment.utc(date, 'MMMM Do, YYYY').toDate();
+
+    const { poster } = document.querySelector('video');
+    if (poster && !/gen/.test(poster)) release.poster = [poster.replace('low', 'hi'), poster];
+
+    const siteId = document.querySelector('#story-and-tags img').src.match(/\w+.jpg/)[0].replace('.jpg', '');
+    const actorsSlug = document.querySelector('h3 a').href.split('/').slice(-2)[0];
+
+    release.photos = Array.from({ length: 5 }, (value, index) => `https://images.psmcdn.net/teamskeet/${siteId}/${actorsSlug}/shared/scenes/new/${String(index + 1).padStart(2, '0')}.jpg`);
+
+    const trailer = document.querySelector('div.right.gray a').href;
+    if (trailer) release.trailer = { src: trailer };
+
+    return release;
+}
+
+async function fetchLatest(site, page = 1) {
+    const url = `https://www.teamskeet.com/t1/updates/load?fltrs[site]=${site.parameters.id}&page=${page}&view=newest&fltrs[time]=ALL&order=DESC`;
+    const res = await bhttp.get(url);
+
+    if (res.statusCode === 200) {
+        return scrapeLatest(res.body.toString(), site);
+    }
+
+    return null;
+}
+
+async function fetchScene(url, site) {
+    const session = bhttp.session(); // resolve redirects
+    const res = await session.get(url);
+
+    return scrapeScene(res.body.toString(), site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchScene,
+};

src/tags.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const logger = require('./logger');
 const knex = require('./knex');
 const whereOr = require('./utils/where-or');
 
@@ -58,7 +59,7 @@ async function matchTags(rawTags) {
 
 async function associateTags(release, releaseId) {
     if (!release.tags || release.tags.length === 0) {
-        console.warn(`No tags available for (${release.site.name}, ${releaseId}) "${release.title}"`);
+        logger.info(`No tags available for (${release.site.name}, ${releaseId}) "${release.title}"`);
         return;
     }