diff --git a/assets/js/networks/actions.js b/assets/js/networks/actions.js index fb97ec76..76a9b173 100644 --- a/assets/js/networks/actions.js +++ b/assets/js/networks/actions.js @@ -21,7 +21,7 @@ function initNetworksActions(store, _router) { sites( orderBy: [PRIORITY_DESC, NAME_ASC], filter: { - enabled: { + show: { equalTo: true, }, }, diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 807c7be8..0e53caac 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -144,7 +144,9 @@ exports.up = knex => Promise.resolve() table.integer('priority', 3) .defaultTo(0); - table.boolean('enabled') + table.boolean('show') + .defaultTo(true); + table.boolean('scrape') .defaultTo(true); table.string('slug', 32) diff --git a/public/img/logos/blowpass/sunlustxxx.png b/public/img/logos/blowpass/sunlustxxx.png new file mode 100644 index 00000000..916f1976 Binary files /dev/null and b/public/img/logos/blowpass/sunlustxxx.png differ diff --git a/seeds/01_sites.js b/seeds/01_sites.js index f4be2bff..3c25c9a2 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -732,6 +732,15 @@ const sites = [ upcoming: '/en/videos/upcoming', }, }, + { + slug: 'sunlustxxx', + name: 'Sun Lust XXX', + url: 'https://www.sunlustxxx.com', + description: '', + network: 'blowpass', + scrape: false, + show: true, // site offline, use only for indexing old scenes + }, // BRAZZERS { slug: 'momsincontrol', @@ -1035,7 +1044,8 @@ const sites = [ description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.', network: 'ddfnetwork', parameters: { native: true }, - enabled: false, // appears to be re-releases only + scrape: false, + show: true, // appears to be re-releases only }, { slug: 'bustylover', @@ -1043,7 +1053,8 @@ const sites = [ url: 'https://bustylover.com', network: 'ddfnetwork', parameters: { native: true }, - enabled: false, // appears to be re-releases only + scrape: false, + show: true, // appears to be re-releases only }, // DIGITAL PLAYGROUND { @@ -1372,7 +1383,8 @@ const sites = [ description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.', network: 'famedigital', parameters: { api: true }, - enabled: false, + scrape: false, + show: false, // no data sources }, { slug: 'peternorth', @@ -3235,7 +3247,8 @@ const sites = [ slug: 'bigboobbundle', url: 'https://www.bigboobbundle.com', network: 'score', - enabled: false, // all content appears to be on subsites + scrape: false, + show: false, // all content appears to be on subsites }, { name: 'Big Boobs POV', @@ -3630,7 +3643,8 @@ const sites = [ url: 'https://www.scorepass.com/scorelandtv', network: 'score', priority: 1, - enabled: false, // appears to be streaming service for other sites + scrape: false, + show: false, // appears to be streaming service for other sites }, { name: 'ScoreTV', @@ -3638,7 +3652,8 @@ const sites = [ url: 'https://www.scoretv.tv', network: 'score', priority: 1, - enabled: false, // similar to or same as Scoreland TV + scrape: false, + show: false, // similar to or same as Scoreland TV }, { name: 'Score Videos', @@ -4079,7 +4094,8 @@ exports.seed = knex => Promise.resolve() parameters: JSON.stringify(site.parameters), network_id: networksMap[site.network], priority: site.priority, - enabled: site.enabled, + scrape: site.scrape, + show: site.show, })); return upsert('sites', sitesWithNetworks, 'slug', knex); diff --git a/src/scrapers/famedigital.js b/src/scrapers/famedigital.js index cf96abad..06369914 100644 --- a/src/scrapers/famedigital.js +++ b/src/scrapers/famedigital.js @@ -1,7 +1,18 @@ 'use strict'; const argv = require('../argv'); -const { fetchLatest, fetchApiLatest, fetchUpcoming, fetchApiUpcoming, fetchScene, fetchProfile, fetchApiProfile } = require('./gamma'); +const { + fetchLatest, + fetchApiLatest, + fetchUpcoming, + fetchApiUpcoming, + fetchScene, + fetchProfile, + fetchApiProfile, + scrapeAll, +} = require('./gamma'); +const { get } = require('../utils/q'); +const slugify = require('../utils/slugify'); function extractLowArtActors(release) { const actors = release.title @@ -47,16 +58,40 @@ function getActorReleasesUrl(actorPath, page = 1) { return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`; } +async function fetchClassicProfile(actorName, siteSlug) { + const actorSlug = slugify(actorName); + + const url = `https://${siteSlug}.com/en/pornstars`; + const { qa } = await get(url); + + const actorPath = qa('option[value*="/pornstar"]') + .find(el => slugify(el.textContent) === actorSlug) + ?.value; + + if (actorPath) { + const actorUrl = `https://${siteSlug}.com${actorPath}`; + const { html } = await get(actorUrl); + + const releases = scrapeAll(html, null, `https://www.${siteSlug}.com`, false); + + return { releases }; + } + + return null; +} + async function networkFetchProfile(actorName) { // not all Fame Digital sites offer Gamma actors - const [devils, rocco, peter] = await Promise.all([ + const [devils, rocco, peter, silvia] = await Promise.all([ fetchApiProfile(actorName, 'devilsfilm', true), fetchApiProfile(actorName, 'roccosiffredi'), argv.withReleases ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl) : [], + argv.withReleases ? fetchClassicProfile(actorName, 'silviasaint') : [], + argv.withReleases ? fetchClassicProfile(actorName, 'silverstonedvd') : [], ]); if (devils || rocco || peter) { - const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || []); + const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []); return { ...peter, diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 7131215a..e5c433b2 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -110,7 +110,7 @@ async function scrapeApiReleases(json, site) { }); } -function scrapeAll(html, site, networkUrl) { +function scrapeAll(html, site, networkUrl, hasTeaser = true) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const scenesElements = $('li[data-itemtype=scene]').toArray(); @@ -146,10 +146,12 @@ function scrapeAll(html, site, networkUrl) { const posterEl = $(element).find('.imgLink img'); if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src'); - release.teaser = { - src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`, - quality: 224, - }; + if (hasTeaser) { + release.teaser = { + src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`, + quality: 224, + }; + } return release; }); @@ -166,7 +168,7 @@ async function scrapeScene(html, url, site) { const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1)); [release.entryId] = new URL(url).pathname.split('/').slice(-1); - release.title = data?.name || videoData.playerOptions.sceneInfos.sceneTitle; + release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name; // date in data object is not the release date of the scene, but the date the entry was added; only use as fallback const dateString = $('.updatedDate').first().text().trim(); diff --git a/src/sites.js b/src/sites.js index a9ec0d7b..44575788 100644 --- a/src/sites.js +++ b/src/sites.js @@ -113,6 +113,7 @@ async function fetchSitesFromArgv() { 'sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', ) + .where('sites.scrape', true) .whereIn('sites.slug', argv.sites || []) .orWhereIn('networks.slug', argv.networks || []) .leftJoin('networks', 'sites.network_id', 'networks.id'); @@ -133,7 +134,7 @@ async function fetchSitesFromConfig() { 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', ) .leftJoin('networks', 'sites.network_id', 'networks.id') - .where('sites.enabled') + .where('sites.scrape', true) .where((builder) => { if (config.include) { builder