From 631ac34573eab397b3bd734acc1c9eefa66ed215 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Fri, 7 Feb 2020 01:06:39 +0100 Subject: [PATCH] Blowpass now uses Gamma module for latest and upcoming. --- seeds/01_sites.js | 20 +++++++++++++ src/scrapers/bangbros.js | 62 ++++++++++++++++------------------------ src/scrapers/blowpass.js | 14 +-------- src/scrapers/gamma.js | 9 ++++-- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 84372fad..f4be2bff 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -683,6 +683,10 @@ const sites = [ url: 'https://www.1000facials.com', description: 'Welcome to 1000Facials.com, your source for the best facial porn with huge cumshots on your favorite teen and MILF pornstars. Watch all the blowjob action inside!', network: 'blowpass', + parameters: { + latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, }, { slug: 'immorallive', @@ -690,6 +694,10 @@ const sites = [ url: 'https://www.immorallive.com', description: 'Watch live sex shows and videos on ImmoralLive.com, featuring wild and crazy sex orgies, group sex, blowjob competitions and toy play from the famous Porno Dan. The hottest pornstars and amateur girls cum hard inside', network: 'blowpass', + parameters: { + latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, }, { slug: 'mommyblowsbest', @@ -697,6 +705,10 @@ const sites = [ url: 'https://www.mommyblowsbest.com', description: 'Welcome to MommyBlowsBest.com. Home to thousands of MILF blowjobs and hot mom porn! Come see why experience counts, right here at MommyBlowsBest.com!', network: 'blowpass', + parameters: { + latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, }, { slug: 'onlyteenblowjobs', @@ -704,6 +716,10 @@ const sites = [ url: 'https://www.onlyteenblowjobs.com', description: 'OnlyTeenBlowjobs.com brings you the best teen blowjob porn featuring today\'s hottest young pornstars and amateurs. Watch as teens use their little mouths to suck and deepthroat the biggest of cocks!', network: 'blowpass', + parameters: { + latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, }, { slug: 'throated', @@ -711,6 +727,10 @@ const sites = [ url: 'https://www.throated.com', description: 'Throated.com is your portal for extreme throat fuck porn, face fucking videos and deepthroat gagging pornstars. Watch teens and MILFs go balls deep, swallowing cock in HD!', network: 'blowpass', + parameters: { + latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, }, // BRAZZERS { diff --git a/src/scrapers/bangbros.js b/src/scrapers/bangbros.js index 8e63b512..49f5ad1f 100644 --- a/src/scrapers/bangbros.js +++ b/src/scrapers/bangbros.js @@ -78,48 +78,36 @@ function scrapeUpcoming(html, site) { } */ -function scrapeScene(html, url, site) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const sceneElement = $('.playerSection'); +function scrapeScene(html, url, _site) { + const { q, qa, qu, qi, qt } = ex(html, '.playerSection'); + const release = {}; - const shootId = sceneElement.find('.vdoCast:contains("Release")').text().replace('Release: ', ''); - const entryId = url.split('/')[3].slice(5); - const title = sceneElement.find('.ps-vdoHdd h1').text(); - const description = sceneElement.find('.vdoDesc').text().trim(); + [release.shootId] = q('.vdoTags + .vdoCast', true).match(/\w+$/); + [release.entryId] = url.split('/')[3].match(/\d+$/); + release.title = q('.ps-vdoHdd h1', true); + release.description = q('.vdoDesc', true); - const [siteName, ...actors] = sceneElement.find('.vdoCast a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - const siteSlug = siteName.replace(/[\s']+/g, '').toLowerCase(); + release.actors = qa('a[href*="/model"]', true); + release.tags = qa('.vdoTags a', true); - const poster = `https:${$('img#player-overlay-image').attr('src')}`; - const trailer = `https:${$('source[type="video/mp4"]').attr('src')}`; + release.stars = Number(q('div[class*="like"]', true).match(/^\d+/)[0]) / 20; - const firstPhotoUrl = `https:${$('img[data-slider-index="1"]').attr('src')}`; - // all scenes seem to have 12 album photos available, not always included on the page - const photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`)); - - const tags = $('.vdoTags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); - - const stars = Number(sceneElement.find('.bVdPl_it_like .bVdPl_txt').text().replace('% like', '')) / 20; - - return { - url, - shootId, - entryId, - title, - description, - actors, - tags, + const poster = qi('img#player-overlay-image'); + release.poster = [ poster, - photos, - trailer: { - src: trailer, - }, - rating: { - stars, - }, - site, - channel: siteSlug === 'bangcasting' ? 'bangbroscasting' : siteSlug, - }; + poster.replace('/big_trailer', '/members/450x340'), // load error fallback + ]; + + release.trailer = { src: qt() }; + + // all scenes seem to have 12 album photos available, not always included on the page + const firstPhotoUrl = ex(html).qi('img[data-slider-index="1"]'); + release.photos = Array.from({ length: 12 }, (val, index) => firstPhotoUrl.replace(/big\d+/, `big${index + 1}`)); + + const [channel] = qu('a[href*="/websites"]').match(/\w+$/); + release.channel = channel === 'bangcasting' ? 'bangbroscasting' : channel; + + return release; } function scrapeProfile(html) { diff --git a/src/scrapers/blowpass.js b/src/scrapers/blowpass.js index 60eb3f99..d89fd9de 100644 --- a/src/scrapers/blowpass.js +++ b/src/scrapers/blowpass.js @@ -2,7 +2,7 @@ const bhttp = require('bhttp'); -const { scrapeAll, scrapeScene, fetchProfile } = require('./gamma'); +const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma'); async function fetchScene(url, site) { // const res = await bhttp.get(url); @@ -21,18 +21,6 @@ async function fetchScene(url, site) { return release; } -async function fetchLatest(site, page = 1) { - const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.slug}/latest/All-Categories/0/All-Pornstars/0/${page}`); - - return scrapeAll(res.body.toString(), site); -} - -async function fetchUpcoming(site) { - const res = await bhttp.get(`https://www.blowpass.com/en/videos/${site.slug}/upcoming`); - - return scrapeAll(res.body.toString(), site); -} - module.exports = { fetchLatest, fetchProfile, diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 0750c23c..1a35e0c4 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -365,11 +365,14 @@ async function fetchLatest(site, page = 1) { const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`; const res = await bhttp.get(url); + console.log(url); + return scrapeAll(res.body.toString(), site); } async function fetchUpcoming(site) { - const res = await bhttp.get(`${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`); + const url = `${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`; + const res = await bhttp.get(url); return scrapeAll(res.body.toString(), site); } @@ -450,10 +453,10 @@ async function fetchApiProfile(actorName, siteSlug) { }); if (res.statusCode === 200 && res.body.results[0].hits.length > 0) { - const actorData = res.body.results[0].hits.find(actor => actor.name === actorName); + const actorData = res.body.results[0].hits.find(actor => slugify(actor.name) === slugify(actorName)); if (actorData) { - const actorScenes = await fetchActorScenes(actorName, apiUrl, siteSlug); + const actorScenes = await fetchActorScenes(actorData.name, apiUrl, siteSlug); return scrapeApiProfile(actorData, actorScenes, siteSlug); }