Added PornPros satellite sites.

This commit is contained in:
ThePendulum 2020-01-15 01:01:16 +01:00
parent 6d5dde774f
commit 5dda81535d
49 changed files with 143 additions and 12 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 7.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.9 KiB

After

Width:  |  Height:  |  Size: 9.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 64 KiB

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

View File

@ -2164,7 +2164,7 @@ function getSites(networksMap) {
}, },
{ {
name: 'Disgraced 18', name: 'Disgraced 18',
slug: 'disgracedeighteen', slug: 'disgraced18',
url: 'https://pornpros.com/site/disgraced18', url: 'https://pornpros.com/site/disgraced18',
network_id: networksMap.pornpros, network_id: networksMap.pornpros,
}, },
@ -2216,6 +2216,84 @@ function getSites(networksMap) {
url: 'https://pornpros.com/site/pimpparade', url: 'https://pornpros.com/site/pimpparade',
network_id: networksMap.pornpros, network_id: networksMap.pornpros,
}, },
{
name: 'Cum 4K',
slug: 'cum4k',
url: 'https://cum4k.com',
network_id: networksMap.pornpros,
},
{
name: 'Tiny 4K',
slug: 'tiny4k',
url: 'https://tiny4k.com',
network_id: networksMap.pornpros,
},
{
name: 'POVD',
slug: 'povd',
url: 'https://povd.com',
network_id: networksMap.pornpros,
},
{
name: 'Lubed',
slug: 'lubed',
url: 'https://lubed.com',
network_id: networksMap.pornpros,
},
{
name: 'Casting Couch X',
slug: 'castingcouchx',
url: 'https://castingcouch-x.com',
network_id: networksMap.pornpros,
},
{
name: 'Passion HD',
slug: 'passionhd',
url: 'https://passion-hd.com',
network_id: networksMap.pornpros,
},
{
name: 'Nanny Spy',
slug: 'nannyspy',
url: 'https://nannyspy.com',
network_id: networksMap.pornpros,
},
{
name: 'Girl Cum',
slug: 'girlcum',
url: 'https://girlcum.com',
network_id: networksMap.pornpros,
},
{
name: 'Pure Mature',
slug: 'puremature',
url: 'https://puremature.com',
network_id: networksMap.pornpros,
},
{
name: 'Fantasy HD',
slug: 'fantasyhd',
url: 'https://fantasyhd.com',
network_id: networksMap.pornpros,
},
{
name: 'Spy Fam',
slug: 'spyfam',
url: 'https://spyfam.com',
network_id: networksMap.pornpros,
},
{
name: 'Exotic 4K',
slug: 'exotic4k',
url: 'https://exotic4k.com',
network_id: networksMap.pornpros,
},
{
name: 'Baeb',
slug: 'baeb',
url: 'https://baeb.com',
network_id: networksMap.pornpros,
},
// PRIVATE // PRIVATE
{ {
slug: 'analintroductions', slug: 'analintroductions',

View File

@ -456,6 +456,12 @@ function getTags(groupsMap) {
alias_for: null, alias_for: null,
group_id: groupsMap.clothing, group_id: groupsMap.clothing,
}, },
{
name: 'fake cum',
slug: 'fake-cum',
alias_for: null,
group_id: groupsMap.finish,
},
{ {
name: 'trainbang', name: 'trainbang',
slug: 'trainbang', slug: 'trainbang',
@ -611,6 +617,11 @@ function getTags(groupsMap) {
slug: 'nipple-clamps', slug: 'nipple-clamps',
alias_for: null, alias_for: null,
}, },
{
name: 'oil',
slug: 'oil',
alias_for: null,
},
{ {
name: 'oral creampie', name: 'oral creampie',
slug: 'oral-creampie', slug: 'oral-creampie',
@ -1581,16 +1592,23 @@ function getSiteTags() {
teenallanal: ['anal', 'mff'], teenallanal: ['anal', 'mff'],
boundgods: ['gay'], boundgods: ['gay'],
buttmachineboys: ['gay'], buttmachineboys: ['gay'],
cum4k: ['fake-cum', 'creampie', '4k'],
deepthroatlove: ['blowjob', 'deepthroat'], deepthroatlove: ['blowjob', 'deepthroat'],
divinebitches: ['femdom'], divinebitches: ['femdom'],
dpparodies: ['parody'], dpparodies: ['parody'],
eighteenyearsold: ['teen'], eighteenyearsold: ['teen'],
exotic4k: ['4k'],
lubed: ['oil'],
familystrokes: ['family'], familystrokes: ['family'],
massagecreep: ['massage'], massagecreep: ['massage'],
menonedge: ['gay'], menonedge: ['gay'],
povd: ['pov'],
puremature: ['milf'],
spyfam: ['family'],
submissived: ['bdsm'], submissived: ['bdsm'],
swallowed: ['blowjob', 'deepthroat', 'facefucking'], swallowed: ['blowjob', 'deepthroat', 'facefucking'],
teenbff: ['mff'], teenbff: ['mff'],
tiny4k: ['4k'],
trueanal: ['anal'], trueanal: ['anal'],
tspussyhunters: ['transsexual'], tspussyhunters: ['transsexual'],
}; };

View File

@ -43,7 +43,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a
.filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
&& moment(release.date).isAfter(afterDate)); // release is older than specified date limit && moment(release.date).isAfter(afterDate)); // release is older than specified date limit
logger.info(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`); logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
if ( if (
uniqueReleases.length > 0 uniqueReleases.length > 0

View File

@ -3,14 +3,18 @@
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
const ex = require('../utils/ex');
function scrapeLatest(html, site) { function scrapeLatest(html, site) {
const { document } = new JSDOM(html).window; const s = ex(html);
const { origin } = new URL(site.url);
return Array.from(document.querySelectorAll('.site-list .card'), (scene) => { const videos = s.qa('.video-releases-list').slice(-1)[0];
return Array.from(videos.querySelectorAll('.card'), (scene) => {
const release = { site }; const release = { site };
release.url = `https://pornpros.com${scene.querySelector(':scope > a').href}`; release.url = `${origin}${scene.querySelector(':scope > a').href}`;
release.entryId = scene.dataset.videoId; release.entryId = scene.dataset.videoId;
release.title = scene.querySelector('.card-title').textContent; release.title = scene.querySelector('.card-title').textContent;
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate(); release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
@ -26,18 +30,25 @@ function scrapeLatest(html, site) {
}); });
} }
function scrapeScene(html, site) { function scrapeScene(html, site, url) {
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
const release = { site }; const release = { site };
const scene = document.querySelector('#t2019-2col'); const scene = document.querySelector('#t2019-2col');
release.url = url;
release.title = scene.querySelector('.t2019-stitle').textContent.trim(); release.title = scene.querySelector('.t2019-stitle').textContent.trim();
release.description = scene.querySelector('#t2019-description').textContent.trim(); release.description = scene.querySelector('#t2019-description').textContent.trim();
release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent); release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent);
const durationEl = scene.querySelector('#t2019-stime'); const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
if (durationEl) release.duration = Number(durationEl.textContent.match(/\d+/)[0]) * 60;
if (durationEls.length > 1) {
release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
} else {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
}
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => `https:${el.src}`); release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => `https:${el.src}`);
@ -54,21 +65,21 @@ function scrapeScene(html, site) {
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const url = `https://pornpros.com/site/${site.slug}?page=${page}`; const url = `${site.url}?page=${page}`;
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeLatest(res.body.toString(), site); return scrapeLatest(res.body.toString(), site);
} }
return null; return [];
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
if (res.statusCode === 200) { if (res.statusCode === 200) {
return scrapeScene(res.body.toString(), site); return scrapeScene(res.body.toString(), site, url);
} }
return null; return null;

View File

@ -60,7 +60,7 @@ function destructConfigNetworks(networks) {
} }
async function findSiteByUrl(url) { async function findSiteByUrl(url) {
const { hostname } = new URL(url); const { hostname, origin } = new URL(url);
const domain = hostname.replace(/www.|tour./, ''); const domain = hostname.replace(/www.|tour./, '');
const sites = await knex('sites') const sites = await knex('sites')
@ -70,6 +70,7 @@ async function findSiteByUrl(url) {
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
) )
.where('sites.url', 'like', `${domain}`) .where('sites.url', 'like', `${domain}`)
.orWhere('sites.url', 'like', `${origin}`)
.orWhere('sites.url', url); .orWhere('sites.url', url);
if (sites.length > 0) { if (sites.length > 0) {

23
src/utils/ex.js Normal file
View File

@ -0,0 +1,23 @@
'use strict';
const { JSDOM } = require('jsdom');
function q(context, selector) {
return context.querySelector(selector);
}
function qa(context, selector) {
return Array.from(context.querySelectorAll(selector));
}
function ex(html) {
const { document } = new JSDOM(html).window;
return {
document,
q: selector => q(document, selector),
qa: selector => qa(document, selector),
};
}
module.exports = ex;