diff --git a/public/img/logos/pornpros/baeb.png b/public/img/logos/pornpros/baeb.png new file mode 100644 index 00000000..ad733763 Binary files /dev/null and b/public/img/logos/pornpros/baeb.png differ diff --git a/public/img/logos/pornpros/castingcouchx.png b/public/img/logos/pornpros/castingcouchx.png new file mode 100644 index 00000000..63773cdb Binary files /dev/null and b/public/img/logos/pornpros/castingcouchx.png differ diff --git a/public/img/logos/pornpros/cockcompetition.png b/public/img/logos/pornpros/cockcompetition.png index 7bae56ae..b9199838 100644 Binary files a/public/img/logos/pornpros/cockcompetition.png and b/public/img/logos/pornpros/cockcompetition.png differ diff --git a/public/img/logos/pornpros/crueltyparty.png b/public/img/logos/pornpros/crueltyparty.png index a699d753..5fb6ee5a 100644 Binary files a/public/img/logos/pornpros/crueltyparty.png and b/public/img/logos/pornpros/crueltyparty.png differ diff --git a/public/img/logos/pornpros/cum4k.png b/public/img/logos/pornpros/cum4k.png new file mode 100644 index 00000000..9c9b98d6 Binary files /dev/null and b/public/img/logos/pornpros/cum4k.png differ diff --git a/public/img/logos/pornpros/cumshotsurprise.png b/public/img/logos/pornpros/cumshotsurprise.png index 6553f999..594cc046 100644 Binary files a/public/img/logos/pornpros/cumshotsurprise.png and b/public/img/logos/pornpros/cumshotsurprise.png differ diff --git a/public/img/logos/pornpros/deepthroatlove.png b/public/img/logos/pornpros/deepthroatlove.png index 97cf92d2..1278a651 100644 Binary files a/public/img/logos/pornpros/deepthroatlove.png and b/public/img/logos/pornpros/deepthroatlove.png differ diff --git a/public/img/logos/pornpros/disgracedeighteen.png b/public/img/logos/pornpros/disgraced18.png similarity index 96% rename from public/img/logos/pornpros/disgracedeighteen.png rename to public/img/logos/pornpros/disgraced18.png index dc48df0f..118f4e0c 100644 Binary files a/public/img/logos/pornpros/disgracedeighteen.png and b/public/img/logos/pornpros/disgraced18.png differ diff --git a/public/img/logos/pornpros/eighteenyearsold.png b/public/img/logos/pornpros/eighteenyearsold.png index 9712045f..ff39d7f8 100644 Binary files a/public/img/logos/pornpros/eighteenyearsold.png and b/public/img/logos/pornpros/eighteenyearsold.png differ diff --git a/public/img/logos/pornpros/eurohumpers.png b/public/img/logos/pornpros/eurohumpers.png index 47ddf903..1184ed53 100644 Binary files a/public/img/logos/pornpros/eurohumpers.png and b/public/img/logos/pornpros/eurohumpers.png differ diff --git a/public/img/logos/pornpros/exotic4k.png b/public/img/logos/pornpros/exotic4k.png new file mode 100644 index 00000000..33800739 Binary files /dev/null and b/public/img/logos/pornpros/exotic4k.png differ diff --git a/public/img/logos/pornpros/fantasyhd.png b/public/img/logos/pornpros/fantasyhd.png new file mode 100644 index 00000000..7623a659 Binary files /dev/null and b/public/img/logos/pornpros/fantasyhd.png differ diff --git a/public/img/logos/pornpros/favicon.png b/public/img/logos/pornpros/favicon.png index 073d6f03..acde575c 100644 Binary files a/public/img/logos/pornpros/favicon.png and b/public/img/logos/pornpros/favicon.png differ diff --git a/public/img/logos/pornpros/fortyozbounce.png b/public/img/logos/pornpros/fortyozbounce.png index e32a910a..bae3fe88 100644 Binary files a/public/img/logos/pornpros/fortyozbounce.png and b/public/img/logos/pornpros/fortyozbounce.png differ diff --git a/public/img/logos/pornpros/freaksofboobs.png b/public/img/logos/pornpros/freaksofboobs.png index cc154efe..ef248e61 100644 Binary files a/public/img/logos/pornpros/freaksofboobs.png and b/public/img/logos/pornpros/freaksofboobs.png differ diff --git a/public/img/logos/pornpros/freaksofcock.png b/public/img/logos/pornpros/freaksofcock.png index cba960a5..619a0bdd 100644 Binary files a/public/img/logos/pornpros/freaksofcock.png and b/public/img/logos/pornpros/freaksofcock.png differ diff --git a/public/img/logos/pornpros/girlcum.png b/public/img/logos/pornpros/girlcum.png new file mode 100644 index 00000000..95234c73 Binary files /dev/null and b/public/img/logos/pornpros/girlcum.png differ diff --git a/public/img/logos/pornpros/jurassiccock.png b/public/img/logos/pornpros/jurassiccock.png index ac174f7d..9a42d1ce 100644 Binary files a/public/img/logos/pornpros/jurassiccock.png and b/public/img/logos/pornpros/jurassiccock.png differ diff --git a/public/img/logos/pornpros/lubed.png b/public/img/logos/pornpros/lubed.png new file mode 100644 index 00000000..db92e613 Binary files /dev/null and b/public/img/logos/pornpros/lubed.png differ diff --git a/public/img/logos/pornpros/massagecreep.png b/public/img/logos/pornpros/massagecreep.png index 49f05b17..206f9c00 100644 Binary files a/public/img/logos/pornpros/massagecreep.png and b/public/img/logos/pornpros/massagecreep.png differ diff --git a/public/img/logos/pornpros/misc/casting-couch-x_original.png b/public/img/logos/pornpros/misc/casting-couch-x_original.png new file mode 100644 index 00000000..f2c98fc4 Binary files /dev/null and b/public/img/logos/pornpros/misc/casting-couch-x_original.png differ diff --git a/public/img/logos/pornpros/misc/exotic-4k_original.png b/public/img/logos/pornpros/misc/exotic-4k_original.png new file mode 100644 index 00000000..f7ac4a3d Binary files /dev/null and b/public/img/logos/pornpros/misc/exotic-4k_original.png differ diff --git a/public/img/logos/pornpros/misc/fantasy-hd_original.png b/public/img/logos/pornpros/misc/fantasy-hd_original.png new file mode 100644 index 00000000..5d0f8873 Binary files /dev/null and b/public/img/logos/pornpros/misc/fantasy-hd_original.png differ diff --git a/public/img/logos/pornpros/misc/girl-cum_original.png b/public/img/logos/pornpros/misc/girl-cum_original.png new file mode 100644 index 00000000..b00ede9b Binary files /dev/null and b/public/img/logos/pornpros/misc/girl-cum_original.png differ diff --git a/public/img/logos/pornpros/misc/logo-f07d09359a.png b/public/img/logos/pornpros/misc/logo-f07d09359a.png new file mode 100644 index 00000000..eb5078e9 Binary files /dev/null and b/public/img/logos/pornpros/misc/logo-f07d09359a.png differ diff --git a/public/img/logos/pornpros/misc/lubed_original.png b/public/img/logos/pornpros/misc/lubed_original.png new file mode 100644 index 00000000..5b6f48c9 Binary files /dev/null and b/public/img/logos/pornpros/misc/lubed_original.png differ diff --git a/public/img/logos/pornpros/misc/nanny-spy_original.png b/public/img/logos/pornpros/misc/nanny-spy_original.png new file mode 100644 index 00000000..ee466648 Binary files /dev/null and b/public/img/logos/pornpros/misc/nanny-spy_original.png differ diff --git a/public/img/logos/pornpros/misc/passion-hd_original.png b/public/img/logos/pornpros/misc/passion-hd_original.png new file mode 100644 index 00000000..d510495e Binary files /dev/null and b/public/img/logos/pornpros/misc/passion-hd_original.png differ diff --git a/public/img/logos/pornpros/misc/povd_original.png b/public/img/logos/pornpros/misc/povd_original.png new file mode 100644 index 00000000..b54b4bac Binary files /dev/null and b/public/img/logos/pornpros/misc/povd_original.png differ diff --git a/public/img/logos/pornpros/misc/pure-mature_original.png b/public/img/logos/pornpros/misc/pure-mature_original.png new file mode 100644 index 00000000..f1b837bc Binary files /dev/null and b/public/img/logos/pornpros/misc/pure-mature_original.png differ diff --git a/public/img/logos/pornpros/misc/spy-fam_original.png b/public/img/logos/pornpros/misc/spy-fam_original.png new file mode 100644 index 00000000..136e5eeb Binary files /dev/null and b/public/img/logos/pornpros/misc/spy-fam_original.png differ diff --git a/public/img/logos/pornpros/misc/tiny4k_original.png b/public/img/logos/pornpros/misc/tiny4k_original.png new file mode 100644 index 00000000..84763e22 Binary files /dev/null and b/public/img/logos/pornpros/misc/tiny4k_original.png differ diff --git a/public/img/logos/pornpros/nannyspy.png b/public/img/logos/pornpros/nannyspy.png new file mode 100644 index 00000000..9c2ee644 Binary files /dev/null and b/public/img/logos/pornpros/nannyspy.png differ diff --git a/public/img/logos/pornpros/network.png b/public/img/logos/pornpros/network.png index 8d269c07..638d0e0b 100644 Binary files a/public/img/logos/pornpros/network.png and b/public/img/logos/pornpros/network.png differ diff --git a/public/img/logos/pornpros/passionhd.png b/public/img/logos/pornpros/passionhd.png new file mode 100644 index 00000000..3cd63142 Binary files /dev/null and b/public/img/logos/pornpros/passionhd.png differ diff --git a/public/img/logos/pornpros/pimpparade.png b/public/img/logos/pornpros/pimpparade.png index 01a8c552..ffefec8b 100644 Binary files a/public/img/logos/pornpros/pimpparade.png and b/public/img/logos/pornpros/pimpparade.png differ diff --git a/public/img/logos/pornpros/povd.png b/public/img/logos/pornpros/povd.png new file mode 100644 index 00000000..e912e3d8 Binary files /dev/null and b/public/img/logos/pornpros/povd.png differ diff --git a/public/img/logos/pornpros/puremature.png b/public/img/logos/pornpros/puremature.png new file mode 100644 index 00000000..107f71fe Binary files /dev/null and b/public/img/logos/pornpros/puremature.png differ diff --git a/public/img/logos/pornpros/realexgirlfriends.png b/public/img/logos/pornpros/realexgirlfriends.png index ba70661f..94af777d 100644 Binary files a/public/img/logos/pornpros/realexgirlfriends.png and b/public/img/logos/pornpros/realexgirlfriends.png differ diff --git a/public/img/logos/pornpros/shadypi.png b/public/img/logos/pornpros/shadypi.png index 3c5d2804..a5e9fc48 100644 Binary files a/public/img/logos/pornpros/shadypi.png and b/public/img/logos/pornpros/shadypi.png differ diff --git a/public/img/logos/pornpros/spyfam.png b/public/img/logos/pornpros/spyfam.png new file mode 100644 index 00000000..72e4fc94 Binary files /dev/null and b/public/img/logos/pornpros/spyfam.png differ diff --git a/public/img/logos/pornpros/teenbff.png b/public/img/logos/pornpros/teenbff.png index aa802234..12d92fd1 100644 Binary files a/public/img/logos/pornpros/teenbff.png and b/public/img/logos/pornpros/teenbff.png differ diff --git a/public/img/logos/pornpros/tiny4k.png b/public/img/logos/pornpros/tiny4k.png new file mode 100644 index 00000000..fe2fa941 Binary files /dev/null and b/public/img/logos/pornpros/tiny4k.png differ diff --git a/seeds/01_sites.js b/seeds/01_sites.js index a44c07cf..4e8b172e 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -2164,7 +2164,7 @@ function getSites(networksMap) { }, { name: 'Disgraced 18', - slug: 'disgracedeighteen', + slug: 'disgraced18', url: 'https://pornpros.com/site/disgraced18', network_id: networksMap.pornpros, }, @@ -2216,6 +2216,84 @@ function getSites(networksMap) { url: 'https://pornpros.com/site/pimpparade', network_id: networksMap.pornpros, }, + { + name: 'Cum 4K', + slug: 'cum4k', + url: 'https://cum4k.com', + network_id: networksMap.pornpros, + }, + { + name: 'Tiny 4K', + slug: 'tiny4k', + url: 'https://tiny4k.com', + network_id: networksMap.pornpros, + }, + { + name: 'POVD', + slug: 'povd', + url: 'https://povd.com', + network_id: networksMap.pornpros, + }, + { + name: 'Lubed', + slug: 'lubed', + url: 'https://lubed.com', + network_id: networksMap.pornpros, + }, + { + name: 'Casting Couch X', + slug: 'castingcouchx', + url: 'https://castingcouch-x.com', + network_id: networksMap.pornpros, + }, + { + name: 'Passion HD', + slug: 'passionhd', + url: 'https://passion-hd.com', + network_id: networksMap.pornpros, + }, + { + name: 'Nanny Spy', + slug: 'nannyspy', + url: 'https://nannyspy.com', + network_id: networksMap.pornpros, + }, + { + name: 'Girl Cum', + slug: 'girlcum', + url: 'https://girlcum.com', + network_id: networksMap.pornpros, + }, + { + name: 'Pure Mature', + slug: 'puremature', + url: 'https://puremature.com', + network_id: networksMap.pornpros, + }, + { + name: 'Fantasy HD', + slug: 'fantasyhd', + url: 'https://fantasyhd.com', + network_id: networksMap.pornpros, + }, + { + name: 'Spy Fam', + slug: 'spyfam', + url: 'https://spyfam.com', + network_id: networksMap.pornpros, + }, + { + name: 'Exotic 4K', + slug: 'exotic4k', + url: 'https://exotic4k.com', + network_id: networksMap.pornpros, + }, + { + name: 'Baeb', + slug: 'baeb', + url: 'https://baeb.com', + network_id: networksMap.pornpros, + }, // PRIVATE { slug: 'analintroductions', diff --git a/seeds/03_tags.js b/seeds/03_tags.js index 20e04fb1..fe539f53 100644 --- a/seeds/03_tags.js +++ b/seeds/03_tags.js @@ -456,6 +456,12 @@ function getTags(groupsMap) { alias_for: null, group_id: groupsMap.clothing, }, + { + name: 'fake cum', + slug: 'fake-cum', + alias_for: null, + group_id: groupsMap.finish, + }, { name: 'trainbang', slug: 'trainbang', @@ -611,6 +617,11 @@ function getTags(groupsMap) { slug: 'nipple-clamps', alias_for: null, }, + { + name: 'oil', + slug: 'oil', + alias_for: null, + }, { name: 'oral creampie', slug: 'oral-creampie', @@ -1581,16 +1592,23 @@ function getSiteTags() { teenallanal: ['anal', 'mff'], boundgods: ['gay'], buttmachineboys: ['gay'], + cum4k: ['fake-cum', 'creampie', '4k'], deepthroatlove: ['blowjob', 'deepthroat'], divinebitches: ['femdom'], dpparodies: ['parody'], eighteenyearsold: ['teen'], + exotic4k: ['4k'], + lubed: ['oil'], familystrokes: ['family'], massagecreep: ['massage'], menonedge: ['gay'], + povd: ['pov'], + puremature: ['milf'], + spyfam: ['family'], submissived: ['bdsm'], swallowed: ['blowjob', 'deepthroat', 'facefucking'], teenbff: ['mff'], + tiny4k: ['4k'], trueanal: ['anal'], tspussyhunters: ['transsexual'], }; diff --git a/src/scrape-sites.js b/src/scrape-sites.js index 15f7e81b..24079ff2 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -43,7 +43,7 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database && moment(release.date).isAfter(afterDate)); // release is older than specified date limit - logger.info(`\x1b[90m${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases\x1b[0m`); + logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`); if ( uniqueReleases.length > 0 diff --git a/src/scrapers/pornpros.js b/src/scrapers/pornpros.js index 6c957ec4..cfdd5a77 100644 --- a/src/scrapers/pornpros.js +++ b/src/scrapers/pornpros.js @@ -3,14 +3,18 @@ const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const moment = require('moment'); +const ex = require('../utils/ex'); function scrapeLatest(html, site) { - const { document } = new JSDOM(html).window; + const s = ex(html); + const { origin } = new URL(site.url); - return Array.from(document.querySelectorAll('.site-list .card'), (scene) => { + const videos = s.qa('.video-releases-list').slice(-1)[0]; + + return Array.from(videos.querySelectorAll('.card'), (scene) => { const release = { site }; - release.url = `https://pornpros.com${scene.querySelector(':scope > a').href}`; + release.url = `${origin}${scene.querySelector(':scope > a').href}`; release.entryId = scene.dataset.videoId; release.title = scene.querySelector('.card-title').textContent; release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate(); @@ -26,18 +30,25 @@ function scrapeLatest(html, site) { }); } -function scrapeScene(html, site) { +function scrapeScene(html, site, url) { const { document } = new JSDOM(html).window; const release = { site }; const scene = document.querySelector('#t2019-2col'); + release.url = url; release.title = scene.querySelector('.t2019-stitle').textContent.trim(); release.description = scene.querySelector('#t2019-description').textContent.trim(); release.actors = Array.from(scene.querySelectorAll('#t2019-models a'), el => el.textContent); - const durationEl = scene.querySelector('#t2019-stime'); - if (durationEl) release.duration = Number(durationEl.textContent.match(/\d+/)[0]) * 60; + const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span')); + + if (durationEls.length > 1) { + release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate(); + release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60; + } else { + release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60; + } release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => `https:${el.src}`); @@ -54,21 +65,21 @@ function scrapeScene(html, site) { } async function fetchLatest(site, page = 1) { - const url = `https://pornpros.com/site/${site.slug}?page=${page}`; + const url = `${site.url}?page=${page}`; const res = await bhttp.get(url); if (res.statusCode === 200) { return scrapeLatest(res.body.toString(), site); } - return null; + return []; } async function fetchScene(url, site) { const res = await bhttp.get(url); if (res.statusCode === 200) { - return scrapeScene(res.body.toString(), site); + return scrapeScene(res.body.toString(), site, url); } return null; diff --git a/src/sites.js b/src/sites.js index ad97de4a..2ec1f9ec 100644 --- a/src/sites.js +++ b/src/sites.js @@ -60,7 +60,7 @@ function destructConfigNetworks(networks) { } async function findSiteByUrl(url) { - const { hostname } = new URL(url); + const { hostname, origin } = new URL(url); const domain = hostname.replace(/www.|tour./, ''); const sites = await knex('sites') @@ -70,6 +70,7 @@ async function findSiteByUrl(url) { 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', ) .where('sites.url', 'like', `${domain}`) + .orWhere('sites.url', 'like', `${origin}`) .orWhere('sites.url', url); if (sites.length > 0) { diff --git a/src/utils/ex.js b/src/utils/ex.js new file mode 100644 index 00000000..c4b1eba2 --- /dev/null +++ b/src/utils/ex.js @@ -0,0 +1,23 @@ +'use strict'; + +const { JSDOM } = require('jsdom'); + +function q(context, selector) { + return context.querySelector(selector); +} + +function qa(context, selector) { + return Array.from(context.querySelectorAll(selector)); +} + +function ex(html) { + const { document } = new JSDOM(html).window; + + return { + document, + q: selector => q(document, selector), + qa: selector => qa(document, selector), + }; +} + +module.exports = ex;