From d2cb74a2525a7864a892b298b595a8420ddc16bd Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sat, 8 Feb 2020 04:52:32 +0100 Subject: [PATCH] Added Fantasy Massage sites. Improved Private scraper, added movie link. --- migrations/20190325001339_releases.js | 4 +- seeds/01_sites.js | 60 +++++++++++++++--- src/media.js | 24 ++++---- src/scrapers/gamma.js | 61 +++++++++++++++--- src/scrapers/private.js | 89 ++++++++------------------- 5 files changed, 143 insertions(+), 95 deletions(-) diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 0e53caac..fba43bb6 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -105,7 +105,7 @@ exports.up = knex => Promise.resolve() table.string('name'); table.string('url'); table.text('description'); - table.string('parameters'); + table.text('parameters'); table.string('slug', 32) .unique(); @@ -140,7 +140,7 @@ exports.up = knex => Promise.resolve() table.string('name'); table.string('url'); table.text('description'); - table.string('parameters'); + table.text('parameters'); table.integer('priority', 3) .defaultTo(0); diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 0f3e13d9..c047ab6d 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -1457,14 +1457,26 @@ const sites = [ }, }, // FANTASY MASSAGE + { + slug: 'fantasymassage', + name: 'Fantasy Massage', + url: 'https://www.fantasymassage.com', + network: 'fantasymassage', + parameters: { + latest: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/Actor/0/upcoming/', + }, + }, { slug: 'allgirlmassage', name: 'All Girl Massage', url: 'https://www.allgirlmassage.com', network: 'fantasymassage', parameters: { - latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', - upcoming: '/en/videos/upcoming', + latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', }, }, { @@ -1473,8 +1485,10 @@ const sites = [ url: 'https://www.nurumassage.com', network: 'fantasymassage', parameters: { - latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', - upcoming: '/en/videos/upcoming', + latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', }, }, { @@ -1483,9 +1497,10 @@ const sites = [ url: 'https://www.trickyspa.com', network: 'fantasymassage', parameters: { - latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', - upcoming: '/en/videos/upcoming', - photos: '/en/photogallery/', + latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', }, }, { @@ -1494,9 +1509,34 @@ const sites = [ url: 'https://www.soapymassage.com', network: 'fantasymassage', parameters: { - latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', - upcoming: '/en/videos/upcoming', - photos: '/en/photogallery/', + latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', + }, + }, + { + slug: 'milkingtable', + name: 'Milking Table', + url: 'https://www.milkingtable.com', + network: 'fantasymassage', + parameters: { + latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', + }, + }, + { + slug: 'massageparlor', + name: 'Massage Parlor', + url: 'https://www.massage-parlor.com', + network: 'fantasymassage', + parameters: { + latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/', + upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/', + deep: 'network', + photos: 'https://www.fantasymassage.com/en/photo/', }, }, // GIRLSWAY diff --git a/src/media.js b/src/media.js index b9a201ea..ac885bb1 100644 --- a/src/media.js +++ b/src/media.js @@ -38,9 +38,15 @@ function pluckPhotos(photos, specifiedLimit) { } async function getEntropy(buffer) { - const { entropy } = await sharp(buffer).stats(); + try { + const { entropy } = await sharp(buffer).stats(); - return entropy; + return entropy; + } catch (error) { + logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`); + + return 7.5; + } } async function createThumbnail(buffer) { @@ -80,7 +86,7 @@ function curatePhotoEntries(files) { })); } -async function findDuplicates(photos, identifier, prop = null, label) { +async function findDuplicates(photos, identifier, prop = null) { const duplicates = await knex('media') .whereIn(identifier, photos.flat().map((photo) => { if (prop) return photo[prop]; @@ -99,14 +105,6 @@ async function findDuplicates(photos, identifier, prop = null, label) { return !duplicateLookup.has((prop && source[prop]) || (source.src && source)); }); - if (duplicates.length > 0) { - logger.info(`${duplicates.length} media items already present by ${identifier} for ${label}`); - } - - if (originals.length > 0) { - logger.info(`Fetching ${originals.length} new media items for ${label}`); - } - return [duplicates, originals]; } @@ -228,6 +226,8 @@ async function storePhotos(photos, { const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label); + logger.info(`Fetching ${sourceOriginals.length} new ${role}s, ${sourceDuplicates.length} already present by source for ${label}`); + const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), { concurrency: 10, }).filter(photo => photo && photo.entropy > entropyFilter); @@ -235,6 +235,8 @@ async function storePhotos(photos, { const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label); + logger.info(`Saving ${hashOriginals.length} new ${role}s, ${hashDuplicates.length} already present by hash for ${label}`); + const savedPhotos = await savePhotos(hashOriginals, { domain, role, diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 1ae11269..a3a37989 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -11,6 +11,16 @@ const logger = require('../logger')(__filename); const { ex, get } = require('../utils/q'); const slugify = require('../utils/slugify'); +function getAlbumUrl(albumPath, site) { + if (site.parameters?.photos) { + return /^http/.test(site.parameters.photos) + ? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}` + : `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`; + } + + return `${site.url}${albumPath}`; +} + async function fetchPhotos(url) { const res = await bhttp.get(url); @@ -25,7 +35,7 @@ function scrapePhotos(html) { if (/\/join|\/createaccount/.test(url)) { // URL links to join page instead of full photo, extract thumbnail - // /createaccount is used by e.g. Tricky Spa + // /createaccount is used by e.g. Tricky Spa native site const src = $(linkEl).find('img').attr('src'); if (src.match('previews/')) { @@ -47,9 +57,9 @@ function scrapePhotos(html) { } async function getPhotos(albumPath, site) { - const albumUrl = site.parameters?.photos - ? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}` - : `${site.url}${albumPath}`; + const albumUrl = getAlbumUrl(albumPath, site); + + console.log(albumUrl); try { const html = await fetchPhotos(albumUrl); @@ -397,28 +407,63 @@ async function fetchApiUpcoming(site) { return fetchApiLatest(site, 1, true); } +function getLatestUrl(site, page) { + if (site.parameters?.latest) { + return /^http/.test(site.parameters.latest) + ? `${site.parameters.latest}${page}` + : `${site.url}${site.parameters.latest}${page}`; + } + + return `${site.url}/en/videos/AllCategories/0/${page}`; +} + +function getUpcomingUrl(site) { + if (site.parameters?.upcoming) { + return /^http/.test(site.parameters.upcoming) + ? `${site.parameters.upcoming}` + : `${site.url}${site.parameters.upcoming}`; + } + + return `${site.url}/en/videos/AllCategories/0/1/upcoming`; +} + async function fetchLatest(site, page = 1) { - const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`; + const url = getLatestUrl(site, page); const res = await bhttp.get(url); return scrapeAll(res.body.toString(), site); } async function fetchUpcoming(site) { - const url = `${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`; + const url = getUpcomingUrl(site); const res = await bhttp.get(url); return scrapeAll(res.body.toString(), site); } +function getDeepUrl(url, site) { + const { pathname } = new URL(url); + + if (site.parameters?.deep === 'network') { + return `${site.network.url}${pathname}`; + } + + if (site.parameters?.deep) { + return `${site.parameters.deep}${pathname}`; + } + + return url; +} + async function fetchScene(url, site, release) { if (site.parameters?.deep === false) { return release; } - const res = await bhttp.get(url); + const deepUrl = getDeepUrl(url, site); + const res = await bhttp.get(deepUrl); - return scrapeScene(res.body.toString(), url, site); + return scrapeScene(res.body.toString(), url, site, deepUrl); } async function fetchActorScenes(actorName, apiUrl, siteSlug) { diff --git a/src/scrapers/private.js b/src/scrapers/private.js index dad97216..7e586304 100644 --- a/src/scrapers/private.js +++ b/src/scrapers/private.js @@ -5,9 +5,6 @@ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); -const fetchSites = require('../sites'); -const { matchTags } = require('../tags'); - async function getPhotos(entryId, site) { const { hostname } = new URL(site.url); @@ -20,27 +17,6 @@ async function getPhotos(entryId, site) { return photos; } -async function getChannelSite($, site) { - const { hostname } = new URL(site.url); - - if (site.isFallback && hostname.match('private.com')) { - const siteElement = $('.content-wrapper .logos-sites a'); - const siteUrl = siteElement.attr('href').slice(0, -1); - const siteName = siteElement.text(); - const siteSlug = siteName.replace(/\s+/g, '').toLowerCase(); - - const channelSite = await fetchSites({ - slug: siteSlug, - name: siteName, - url: siteUrl, - }); - - return channelSite; - } - - return site; -} - function scrapeLatest(html, site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const sceneElements = $('.content-wrapper .scene').toArray(); @@ -82,55 +58,40 @@ function scrapeLatest(html, site) { async function scrapeScene(html, url, site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); + const release = {}; - const entryId = url.split('/').slice(-1)[0]; - const title = $('.video-wrapper meta[itemprop="name"]').attr('content'); + [release.entryId] = url.split('/').slice(-1); + release.title = $('.video-wrapper meta[itemprop="name"]').attr('content'); + release.description = $('.video-wrapper meta[itemprop="description"]').attr('content'); - const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate(); - const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); + release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate(); + release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - const description = $('.video-wrapper meta[itemprop="description"]').attr('content'); - const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g); - const duration = Number(minutes) * 60 + Number(seconds); + const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content'); + + if (timestamp) { + const [minutes, seconds] = timestamp.match(/\d+/g); + release.duration = Number(minutes) * 60 + Number(seconds); + } + + release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); + release.likes = Number($('.content-desc #social-actions #likes').text()); const posterScript = $('script:contains(poster)').html(); const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4); - const poster = $('meta[property="og:image"]').attr('content') || posterLink; + release.poster = $('meta[property="og:image"]').attr('content') || posterLink; - const trailerElementSrc = $('#videojs-trailer source').attr('src'); - const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc; - - const likes = Number($('.content-desc #social-actions #likes').text()); - - const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); - - const [tags, photos, channelSite] = await Promise.all([ - matchTags(rawTags), - getPhotos(entryId, site), - getChannelSite($, site), - ]); - - const scene = { - url, - entryId, - title, - date, - actors, - description, - duration, - tags, - poster, - photos, - trailer: { - src: trailer, - }, - rating: { - likes, - }, - site: channelSite || site, + release.trailer = { + src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'), }; - return scene; + release.photos = await getPhotos(release.entryId, site); + release.movie = $('a[data-track="FULL MOVIE"]').attr('href'); + + const siteElement = $('.content-wrapper .logos-sites a'); + if (siteElement) release.channel = siteElement.text().replace(/\s+/g, '').toLowerCase(); + + return release; } async function fetchLatest(site, page = 1) {