From 1cba51fbfdd7586570249e86497ca8a6676ce5fb Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 9 Jul 2023 05:08:35 +0200 Subject: [PATCH] Moved Blowpass to Gamma API. --- seeds/02_sites.js | 3 +- src/scrapers/blowpass.js | 27 -------- src/scrapers/dogfart.js | 135 --------------------------------------- src/scrapers/scrapers.js | 7 -- src/updates.js | 2 + 5 files changed, 3 insertions(+), 171 deletions(-) delete mode 100755 src/scrapers/blowpass.js delete mode 100755 src/scrapers/dogfart.js diff --git a/seeds/02_sites.js b/seeds/02_sites.js index b3987704..f2fd1a74 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -1959,8 +1959,7 @@ const sites = [ description: 'Throated.com is your portal for extreme throat fuck porn, face fucking videos and deepthroat gagging pornstars. Watch teens and MILFs go balls deep, swallowing cock in HD!', parent: 'blowpass', parameters: { - latest: '/en/videos/latest/All-Categories/0/All-Pornstars/0/', - upcoming: '/en/videos/upcoming', + layout: 'api', }, }, { diff --git a/src/scrapers/blowpass.js b/src/scrapers/blowpass.js deleted file mode 100755 index 47565c94..00000000 --- a/src/scrapers/blowpass.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -const { fetchScene, fetchLatest, fetchUpcoming, fetchProfile } = require('./gamma'); - -async function fetchSceneWrapper(url, site, baseRelease, options) { - const release = await fetchScene(url, site, baseRelease, options); - - if (site.isNetwork && release.channel) { - const channelUrl = url.replace('blowpass.com', `${release.channel}.com`); - - if (['onlyteenblowjobs', 'mommyblowsbest'].includes(release.channel)) { - release.url = channelUrl.replace(/video\/\w+\//, 'scene/'); - return release; - } - - release.url = channelUrl.replace(/video\/\w+\//, 'video/'); - } - - return release; -} - -module.exports = { - fetchLatest, - fetchProfile, - fetchUpcoming, - fetchScene: fetchSceneWrapper, -}; diff --git a/src/scrapers/dogfart.js b/src/scrapers/dogfart.js deleted file mode 100755 index 7a11709f..00000000 --- a/src/scrapers/dogfart.js +++ /dev/null @@ -1,135 +0,0 @@ -'use strict'; - -const slugify = require('../utils/slugify'); -const qu = require('../utils/qu'); - -async function getPhotos(albumUrl, channel) { - const res = await qu.get(albumUrl); - - if (!res.ok) { - return []; - } - - const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1); - const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10); - - const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => { - const pageUrl = `${channel.url}${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`; - const networkPageUrl = `https://dogfartnetwork.com${lastPhotoPage.replace('tourx', 'tour').replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`; - - const extract = ({ query }) => query.img('.scenes-module img'); - - return [ - { - url: pageUrl, - extract, - }, - { - url: networkPageUrl, - extract, - }, - ]; - }); - - return photoUrls; -} - -function scrapeLatest(scenes, site, filter = true) { - return scenes.reduce((acc, { query }) => { - const release = {}; - - const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name'); - - release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url }); - release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`; - - release.title = query.cnt('.scene-title'); - // release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com - - // release.poster = `https:${element.querySelector('img').src}`; - release.poster = query.img(); - release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url'); - - release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase(); - - if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) { - // different dogfart site - return { ...acc, unextracted: [...acc.unextracted, release] }; - } - - return { ...acc, scenes: [...acc.scenes, release] }; - }, { - scenes: [], - unextracted: [], - }); -} - -async function scrapeScene({ query }, url, channel, baseScene, parameters) { - const release = {}; - const { origin, pathname } = new URL(url); - - release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase(); - release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`; - - release.title = query.cnt('.description-title') || query.text('.scene-title'); - release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({ - name: query.cnt(actorEl), - url: query.url(actorEl, null, 'href', { origin: channel.url }), - })); - - release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description, [itemprop="description"]')?.replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim(); - - release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content'); - release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container'); - - release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/); - - const trailer = query.video('.html5-video', 'data-trailer'); - const lastPhotosUrl = query.urls('.pagination a').at(-1); - - release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image'); - - if (trailer && !trailer?.includes('join')) { - release.trailer = trailer; - } - - if (lastPhotosUrl && parameters.includePhotos) { - release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url); - } - - release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2)); - - return release; -} - -async function fetchLatest(channel, page = 1, { parameters }) { - const res = await qu.getAll(parameters.latest ? `${parameters.latest}/?p=${page}` : `${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container'); - - if (res.ok) { - return scrapeLatest(res.items, channel); - } - - return res.status; -} - -async function fetchProfile(baseActor, entity) { - const slug = slugify(baseActor.name, '+'); - const url = `https://www.dogfartnetwork.com/tour/girls/${slug}/`; - - const res = await qu.getAll(url, '.recent-updates'); - - if (res.ok) { - const { scenes } = scrapeLatest(res.items, entity, false); - - // no bio available - return { scenes }; - } - - return res.status; -} - -module.exports = { - fetchLatest, - fetchProfile, - scrapeScene, -}; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index d0828d34..45c30402 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -10,13 +10,11 @@ const badoink = require('./badoink'); const bamvisions = require('./bamvisions'); const bang = require('./bang'); const bangbros = require('./bangbros'); -const blowpass = require('./blowpass'); const bradmontana = require('./bradmontana'); const cherrypimps = require('./cherrypimps'); const cumlouder = require('./cumlouder'); const czechav = require('./czechav'); const ddfnetwork = require('./ddfnetwork'); -// const dogfart = require('./dogfart'); const dorcel = require('./dorcel'); const fabulouscash = require('./fabulouscash'); const famedigital = require('./famedigital'); @@ -87,15 +85,12 @@ const scrapers = { bamvisions, bang, bangbros, - blowpass, bradmontana, brazzers: mindgeek, cherrypimps, cumlouder, czechav, pornworld: ddfnetwork, - // dogfart, - // dogfartnetwork: dogfart, dorcel, elegantangel: adultempire, famedigital, @@ -189,7 +184,6 @@ const scrapers = { blacked: vixen, blackedraw: vixen, blackambush: elevatedx, - blowpass, bluedonkeymedia, meidenvanholland: bluedonkeymedia, vurigvlaanderen: bluedonkeymedia, @@ -204,7 +198,6 @@ const scrapers = { deeplush: nubiles, devilsfilm: famedigital, digitalplayground: mindgeek, - // dogfartnetwork: dogfart, dorcelclub: dorcel, doubleviewcasting: firstanalquest, dtfsluts: fullpornnetwork, diff --git a/src/updates.js b/src/updates.js index 45654a21..07d4d375 100755 --- a/src/updates.js +++ b/src/updates.js @@ -229,6 +229,8 @@ async function scrapeChannel(channelEntity, accNetworkReleases, beforeNetwork) { const scraper = resolveScraper(channelEntity); const layoutScraper = resolveLayoutScraper(channelEntity, scraper); + console.log(layoutScraper); + if (!layoutScraper) { logger.warn(`No scraper found for '${channelEntity.name}' (${channelEntity.parent?.name})`); return emptyReleases;