Added profile releases for classic Fame Digital sites (Silvia Saint and Silverstone DVD).

assets/js/networks/actions.js

@@ -21,7 +21,7 @@ function initNetworksActions(store, _router) {
                     sites(
                         orderBy: [PRIORITY_DESC, NAME_ASC],
                         filter: {
-                            enabled: {
+                            show: {
                                 equalTo: true,
                             },
                         },

migrations/20190325001339_releases.js

@@ -144,7 +144,9 @@ exports.up = knex => Promise.resolve()
 
         table.integer('priority', 3)
             .defaultTo(0);
-        table.boolean('enabled')
+        table.boolean('show')
+            .defaultTo(true);
+        table.boolean('scrape')
             .defaultTo(true);
 
         table.string('slug', 32)

seeds/01_sites.js

@@ -732,6 +732,15 @@ const sites = [
             upcoming: '/en/videos/upcoming',
         },
     },
+    {
+        slug: 'sunlustxxx',
+        name: 'Sun Lust XXX',
+        url: 'https://www.sunlustxxx.com',
+        description: '',
+        network: 'blowpass',
+        scrape: false,
+        show: true, // site offline, use only for indexing old scenes
+    },
     // BRAZZERS
     {
         slug: 'momsincontrol',
@@ -1035,7 +1044,8 @@ const sites = [
         description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
         network: 'ddfnetwork',
         parameters: { native: true },
-        enabled: false, // appears to be re-releases only
+        scrape: false,
+        show: true, // appears to be re-releases only
     },
     {
         slug: 'bustylover',
@@ -1043,7 +1053,8 @@ const sites = [
         url: 'https://bustylover.com',
         network: 'ddfnetwork',
         parameters: { native: true },
-        enabled: false, // appears to be re-releases only
+        scrape: false,
+        show: true, // appears to be re-releases only
     },
     // DIGITAL PLAYGROUND
     {
@@ -1372,7 +1383,8 @@ const sites = [
         description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
         network: 'famedigital',
         parameters: { api: true },
-        enabled: false,
+        scrape: false,
+        show: false, // no data sources
     },
     {
         slug: 'peternorth',
@@ -3235,7 +3247,8 @@ const sites = [
         slug: 'bigboobbundle',
         url: 'https://www.bigboobbundle.com',
         network: 'score',
-        enabled: false, // all content appears to be on subsites
+        scrape: false,
+        show: false, // all content appears to be on subsites
     },
     {
         name: 'Big Boobs POV',
@@ -3630,7 +3643,8 @@ const sites = [
         url: 'https://www.scorepass.com/scorelandtv',
         network: 'score',
         priority: 1,
-        enabled: false, // appears to be streaming service for other sites
+        scrape: false,
+        show: false, // appears to be streaming service for other sites
     },
     {
         name: 'ScoreTV',
@@ -3638,7 +3652,8 @@ const sites = [
         url: 'https://www.scoretv.tv',
         network: 'score',
         priority: 1,
-        enabled: false, // similar to or same as Scoreland TV
+        scrape: false,
+        show: false, // similar to or same as Scoreland TV
     },
     {
         name: 'Score Videos',
@@ -4079,7 +4094,8 @@ exports.seed = knex => Promise.resolve()
             parameters: JSON.stringify(site.parameters),
             network_id: networksMap[site.network],
             priority: site.priority,
-            enabled: site.enabled,
+            scrape: site.scrape,
+            show: site.show,
         }));
 
         return upsert('sites', sitesWithNetworks, 'slug', knex);

src/scrapers/famedigital.js

@@ -1,7 +1,18 @@
 'use strict';
 
 const argv = require('../argv');
-const { fetchLatest, fetchApiLatest, fetchUpcoming, fetchApiUpcoming, fetchScene, fetchProfile, fetchApiProfile } = require('./gamma');
+const {
+    fetchLatest,
+    fetchApiLatest,
+    fetchUpcoming,
+    fetchApiUpcoming,
+    fetchScene,
+    fetchProfile,
+    fetchApiProfile,
+    scrapeAll,
+} = require('./gamma');
+const { get } = require('../utils/q');
+const slugify = require('../utils/slugify');
 
 function extractLowArtActors(release) {
     const actors = release.title
@@ -47,16 +58,40 @@ function getActorReleasesUrl(actorPath, page = 1) {
     return `https://www.peternorth.com/en/videos/All-Categories/0${actorPath}/All-Dvds/0/latest/${page}`;
 }
 
+async function fetchClassicProfile(actorName, siteSlug) {
+    const actorSlug = slugify(actorName);
+
+    const url = `https://${siteSlug}.com/en/pornstars`;
+    const { qa } = await get(url);
+
+    const actorPath = qa('option[value*="/pornstar"]')
+        .find(el => slugify(el.textContent) === actorSlug)
+        ?.value;
+
+    if (actorPath) {
+        const actorUrl = `https://${siteSlug}.com${actorPath}`;
+        const { html } = await get(actorUrl);
+
+        const releases = scrapeAll(html, null, `https://www.${siteSlug}.com`, false);
+
+        return { releases };
+    }
+
+    return null;
+}
+
 async function networkFetchProfile(actorName) {
     // not all Fame Digital sites offer Gamma actors
-    const [devils, rocco, peter] = await Promise.all([
+    const [devils, rocco, peter, silvia] = await Promise.all([
         fetchApiProfile(actorName, 'devilsfilm', true),
         fetchApiProfile(actorName, 'roccosiffredi'),
         argv.withReleases ? fetchProfile(actorName, 'peternorth', true, getActorReleasesUrl) : [],
+        argv.withReleases ? fetchClassicProfile(actorName, 'silviasaint') : [],
+        argv.withReleases ? fetchClassicProfile(actorName, 'silverstonedvd') : [],
     ]);
 
     if (devils || rocco || peter) {
-        const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || []);
+        const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || [], silvia?.releases || []);
 
         return {
             ...peter,

src/scrapers/gamma.js

@@ -110,7 +110,7 @@ async function scrapeApiReleases(json, site) {
     });
 }
 
-function scrapeAll(html, site, networkUrl) {
+function scrapeAll(html, site, networkUrl, hasTeaser = true) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
     const scenesElements = $('li[data-itemtype=scene]').toArray();
 
@@ -146,10 +146,12 @@ function scrapeAll(html, site, networkUrl) {
         const posterEl = $(element).find('.imgLink img');
         if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
 
+        if (hasTeaser) {
             release.teaser = {
                 src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`,
                 quality: 224,
             };
+        }
 
         return release;
     });
@@ -166,7 +168,7 @@ async function scrapeScene(html, url, site) {
     const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
 
     [release.entryId] = new URL(url).pathname.split('/').slice(-1);
-    release.title = data?.name || videoData.playerOptions.sceneInfos.sceneTitle;
+    release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
 
     // date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
     const dateString = $('.updatedDate').first().text().trim();

src/sites.js

@@ -113,6 +113,7 @@ async function fetchSitesFromArgv() {
             'sites.*',
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
+        .where('sites.scrape', true)
         .whereIn('sites.slug', argv.sites || [])
         .orWhereIn('networks.slug', argv.networks || [])
         .leftJoin('networks', 'sites.network_id', 'networks.id');
@@ -133,7 +134,7 @@ async function fetchSitesFromConfig() {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .leftJoin('networks', 'sites.network_id', 'networks.id')
-        .where('sites.enabled')
+        .where('sites.scrape', true)
         .where((builder) => {
             if (config.include) {
                 builder