Added Team Skeet scraper.

This commit is contained in:
ThePendulum 2020-01-12 05:30:46 +01:00
parent bce2c73cbe
commit 97cb4a061e
46 changed files with 352 additions and 8 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 106 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 865 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 115 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 182 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 50 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 70 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 91 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 73 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

View File

@ -120,6 +120,12 @@ const networks = [
url: 'https://www.realitykings.com', url: 'https://www.realitykings.com',
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com', description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
}, },
{
slug: 'teamskeet',
name: 'Team Skeet',
url: 'https://www.teamskeet.com',
description: 'Welcome to teamskeet.com, the largest collection of exclusive teen porn sites and videos on the web. Check out our TeamSkeet porn sites now.',
},
{ {
slug: 'vixen', slug: 'vixen',
name: 'Vixen', name: 'Vixen',

View File

@ -2480,6 +2480,230 @@ function getSites(networksMap) {
slug: 'monstercurves', slug: 'monstercurves',
network_id: networksMap.realitykings, network_id: networksMap.realitykings,
}, },
{
slug: 'exxxtrasmall',
name: 'Exxxtra Small',
description: '',
url: 'https://www.exxxtrasmall.com',
parameters: JSON.stringify({ id: 'exs' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teenpies',
name: 'Teen Pies',
description: '',
url: 'https://www.teenpies.com',
parameters: JSON.stringify({ id: 'tp' }),
network_id: networksMap.teamskeet,
},
{
slug: 'innocenthigh',
name: 'Innocent High',
description: '',
url: 'https://www.innocenthigh.com',
parameters: JSON.stringify({ id: 'ih' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teencurves',
name: 'Teen Curves',
description: '',
url: 'https://www.teencurves.com',
parameters: JSON.stringify({ id: 'tc' }),
network_id: networksMap.teamskeet,
},
{
slug: 'cfnmteens',
name: 'CFNM Teens',
description: '',
url: 'https://www.cfnmteens.com',
parameters: JSON.stringify({ id: 'cfnm' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teensloveanal',
name: 'Teens Love Anal',
description: '',
url: 'https://www.teensloveanal.com',
parameters: JSON.stringify({ id: 'tla' }),
network_id: networksMap.teamskeet,
},
{
slug: 'mybabysittersclub',
name: 'My Babysitters Club',
description: '',
url: 'https://www.mybabysittersclub.com',
parameters: JSON.stringify({ id: 'bsc' }),
network_id: networksMap.teamskeet,
},
{
slug: 'shesnew',
name: 'She\'s New',
description: '',
url: 'https://www.shesnow.com',
parameters: JSON.stringify({ id: 'bsc' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teensdoporn',
name: 'Teens Do Porn',
description: '',
url: 'https://www.teensdoporn.com',
parameters: JSON.stringify({ id: 'tdp' }),
network_id: networksMap.teamskeet,
},
{
slug: 'povlife',
name: 'POV Life',
description: '',
url: 'https://www.povlife.com',
parameters: JSON.stringify({ id: 'pov' }),
network_id: networksMap.teamskeet,
},
{
slug: 'therealworkout',
name: 'The Real Workout',
description: '',
url: 'https://www.therealworkout.com',
parameters: JSON.stringify({ id: 'trw' }),
network_id: networksMap.teamskeet,
},
{
slug: 'thisgirlsucks',
name: 'This Girl Sucks',
description: '',
url: 'https://www.thisgirlsucks.com',
parameters: JSON.stringify({ id: 'tgs' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teenslovemoney',
name: 'Teens Love Money',
description: '',
url: 'https://www.teenslovemoney.com',
parameters: JSON.stringify({ id: 'tlm' }),
network_id: networksMap.teamskeet,
},
{
slug: 'oyeloca',
name: 'Oye Loca',
description: '',
url: 'https://www.oyeloca.com',
parameters: JSON.stringify({ id: 'ol' }),
network_id: networksMap.teamskeet,
},
{
slug: 'tittyattack',
name: 'Titty Attack',
description: '',
url: 'https://www.tittyattack.com',
parameters: JSON.stringify({ id: 'ta' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teenyblack',
name: 'Teeny Black',
description: '',
url: 'https://www.teenyblack.com',
parameters: JSON.stringify({ id: 'tb' }),
network_id: networksMap.teamskeet,
},
{
slug: 'lusthd',
name: 'Lust HD',
description: '',
url: 'https://www.lusthd.com',
parameters: JSON.stringify({ id: 'lhd' }),
network_id: networksMap.teamskeet,
},
{
slug: 'rubateen',
name: 'Rub A Teen',
description: '',
url: 'https://www.rubateen.com',
parameters: JSON.stringify({ id: 'rat' }),
network_id: networksMap.teamskeet,
},
{
slug: 'herfreshmanyear',
name: 'Her Freshman Year',
description: '',
url: 'https://www.exxxtrasmall.com',
parameters: JSON.stringify({ id: 'hfy' }),
network_id: networksMap.teamskeet,
},
{
slug: 'selfdesire',
name: 'Self Desire',
description: '',
url: 'https://www.selfdesire.com',
parameters: JSON.stringify({ id: 'sd' }),
network_id: networksMap.teamskeet,
},
{
slug: 'solointerviews',
name: 'Solo Interviews',
description: '',
url: 'https://www.solointerviews.com',
parameters: JSON.stringify({ id: 'si' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teamskeetextras',
name: 'Team Skeet Extras',
description: '',
url: 'https://www.teamskeetextras.com',
parameters: JSON.stringify({ id: 'tse' }),
network_id: networksMap.teamskeet,
},
{
slug: 'dyked',
name: 'Dyked',
description: '',
url: 'https://www.dyked.com',
parameters: JSON.stringify({ id: 'dyk' }),
network_id: networksMap.teamskeet,
},
{
slug: 'badmilfs',
name: 'Bad MILFs',
description: '',
url: 'https://www.badmilfs.com',
parameters: JSON.stringify({ id: 'bad' }),
network_id: networksMap.teamskeet,
},
{
slug: 'gingerpatch',
name: 'Ginger Patch',
description: '',
url: 'https://www.gingerpatch.com',
parameters: JSON.stringify({ id: 'gp' }),
network_id: networksMap.teamskeet,
},
{
slug: 'bracefaced',
name: 'Brace Faced',
description: '',
url: 'https://www.bracefaced.com',
parameters: JSON.stringify({ id: 'bfd' }),
network_id: networksMap.teamskeet,
},
{
slug: 'teenjoi',
name: 'Teen JOI',
description: '',
url: 'https://www.teenjoi.com',
parameters: JSON.stringify({ id: 'joi' }),
network_id: networksMap.teamskeet,
},
{
slug: 'stepsiblings',
name: 'Step Siblings',
description: '',
url: 'https://www.stepsiblings.com',
parameters: JSON.stringify({ id: 'sss' }),
network_id: networksMap.teamskeet,
},
// VIXEN // VIXEN
{ {
slug: 'vixen', slug: 'vixen',

View File

@ -4,6 +4,7 @@
const Promise = require('bluebird'); const Promise = require('bluebird');
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const logger = require('../logger');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
async function fetchToken(site) { async function fetchToken(site) {
@ -30,12 +31,17 @@ async function fetchActors(entryId, site, { token, time }) {
async function fetchTrailerLocation(entryId, site) { async function fetchTrailerLocation(entryId, site) {
const url = `${site.url}/api/download/${entryId}/hd1080/stream`; const url = `${site.url}/api/download/${entryId}/hd1080/stream`;
const res = await bhttp.get(url, {
followRedirects: false,
});
if (res.statusCode === 302) { try {
return res.headers.location; const res = await bhttp.get(url, {
followRedirects: false,
});
if (res.statusCode === 302) {
return res.headers.location;
}
} catch (error) {
logger.warn(`${site.name}: Unable to fetch trailer at '${url}': ${error.message}`);
} }
return null; return null;
@ -72,11 +78,11 @@ function scrapeLatest(scenes, site, tokens) {
return Promise.map(scenes, async scene => scrapeScene(scene, site, tokens), { concurrency: 10 }); return Promise.map(scenes, async scene => scrapeScene(scene, site, tokens), { concurrency: 10 });
} }
async function fetchLatest(site) { async function fetchLatest(site, page = 1) {
const { time, token } = await fetchToken(site); const { time, token } = await fetchToken(site);
// transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory // transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory
const url = `${site.url}/sapi/${token}/${time}/content.load?limit=20&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`; const url = `${site.url}/sapi/${token}/${time}/content.load?limit=50&offset=${(page - 1) * 50}&transitParameters[v1]=OhUOlmasXD&transitParameters[v2]=OhUOlmasXD&transitParameters[preset]=videos`;
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (res.statusCode === 200 && res.body.status) { if (res.statusCode === 200 && res.body.status) {

View File

@ -15,6 +15,7 @@ const pervcity = require('./pervcity');
const privateNetwork = require('./private'); // reserved keyword const privateNetwork = require('./private'); // reserved keyword
const naughtyamerica = require('./naughtyamerica'); const naughtyamerica = require('./naughtyamerica');
const realitykings = require('./realitykings'); const realitykings = require('./realitykings');
const teamskeet = require('./teamskeet');
const vixen = require('./vixen'); const vixen = require('./vixen');
// releases and profiles // releases and profiles
@ -52,6 +53,7 @@ module.exports = {
private: privateNetwork, private: privateNetwork,
naughtyamerica, naughtyamerica,
realitykings, realitykings,
teamskeet,
vixen, vixen,
xempire, xempire,
}, },

105
src/scrapers/teamskeet.js Normal file
View File

@ -0,0 +1,105 @@
'use strict';
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');
function extractTitle(pathname) {
return pathname
.split('/')
.slice(-2)[0]
.split('_')
.map(seg => `${seg.charAt(0).toUpperCase()}${seg.slice(1)}`)
.join(' ');
}
function extractActors(str) {
return str
.split(/,|\band/)
.filter(actor => !/\.{3}/.test(actor))
.map(actor => actor.trim())
.filter(actor => actor.length > 0);
}
function scrapeLatest(html, site) {
const { document } = new JSDOM(html).window;
const scenes = Array.from(document.querySelectorAll('#updatesList li.grey, #updatesList li.white'));
return scenes.map((scene) => {
const release = { site };
const link = scene.querySelector('.info a');
const poster = scene.querySelector('img');
const { pathname } = new URL(link);
[release.entryId] = poster.id.match(/\d+/);
release.url = `https://www.teamskeet.com${pathname}`;
release.title = extractTitle(pathname);
release.date = moment.utc(scene.querySelector('strong').textContent, 'MM/DD/YYYY').toDate();
const photos = Array.from({ length: 5 }, (_value, index) => poster.dataset.original.replace(/\d+.jpg/, `${String(index + 1).padStart(2, '0')}.jpg`));
[release.poster] = photos;
release.photos = photos.slice(1);
const actors = scene.querySelector('div span[rel="test"]').textContent;
release.actors = extractActors(actors);
return release;
});
}
function scrapeScene(html, site) {
const { document } = new JSDOM(html).window;
const release = { site };
release.entryId = document.querySelector('#story-and-tags .scene_rater').attributes.rel.value;
release.description = document.querySelector('#story-and-tags td:nth-child(2) div').textContent;
const [actors, title, channel] = document.querySelector('title').textContent.split('|').map(item => item.trim());
release.title = title;
release.actors = extractActors(actors);
release.channel = channel.toLowerCase();
release.tags = Array.from(document.querySelectorAll('#story-and-tags tr:nth-child(2) a'), el => el.rel);
const date = document.querySelector('h3 ~ div:nth-child(4), h3 ~ div div.gray:not(.scene_rater)').textContent.split(':')[1].trim();
release.date = moment.utc(date, 'MMMM Do, YYYY').toDate();
const { poster } = document.querySelector('video');
if (poster && !/gen/.test(poster)) release.poster = [poster.replace('low', 'hi'), poster];
const siteId = document.querySelector('#story-and-tags img').src.match(/\w+.jpg/)[0].replace('.jpg', '');
const actorsSlug = document.querySelector('h3 a').href.split('/').slice(-2)[0];
release.photos = Array.from({ length: 5 }, (value, index) => `https://images.psmcdn.net/teamskeet/${siteId}/${actorsSlug}/shared/scenes/new/${String(index + 1).padStart(2, '0')}.jpg`);
const trailer = document.querySelector('div.right.gray a').href;
if (trailer) release.trailer = { src: trailer };
return release;
}
async function fetchLatest(site, page = 1) {
const url = `https://www.teamskeet.com/t1/updates/load?fltrs[site]=${site.parameters.id}&page=${page}&view=newest&fltrs[time]=ALL&order=DESC`;
const res = await bhttp.get(url);
if (res.statusCode === 200) {
return scrapeLatest(res.body.toString(), site);
}
return null;
}
async function fetchScene(url, site) {
const session = bhttp.session(); // resolve redirects
const res = await session.get(url);
return scrapeScene(res.body.toString(), site);
}
module.exports = {
fetchLatest,
fetchScene,
};

View File

@ -1,5 +1,6 @@
'use strict'; 'use strict';
const logger = require('./logger');
const knex = require('./knex'); const knex = require('./knex');
const whereOr = require('./utils/where-or'); const whereOr = require('./utils/where-or');
@ -58,7 +59,7 @@ async function matchTags(rawTags) {
async function associateTags(release, releaseId) { async function associateTags(release, releaseId) {
if (!release.tags || release.tags.length === 0) { if (!release.tags || release.tags.length === 0) {
console.warn(`No tags available for (${release.site.name}, ${releaseId}) "${release.title}"`); logger.info(`No tags available for (${release.site.name}, ${releaseId}) "${release.title}"`);
return; return;
} }