Added parent-child relations to network, showing parent network in sidebar. Added Burning Angel using Gamma API.

assets/components/networks/network.vue

@@ -7,10 +7,10 @@
 
         <div
             class="network"
-            :class="{ nosites: sites.length === 0 }"
+            :class="{ nosites: sites.length === 0 && networks.length === 0 }"
         >
             <div
-                v-show="sites.length > 0"
+                v-show="sites.length > 0 || networks.length > 0"
                 class="sidebar"
                 :class="{ expanded }"
             >
@@ -37,9 +37,21 @@
                     :sites="sites"
                     :class="{ expanded }"
                 />
+
+                <Network
+                    v-for="childNetwork in networks"
+                    :key="`network-${childNetwork.id}`"
+                    :network="childNetwork"
+                />
+
+                <Network
+                    v-if="network.parent"
+                    :network="network.parent"
+                    class="parent"
+                />
             </div>
 
-            <template v-if="sites.length > 0">
+            <template v-if="sites.length > 0 || networks.length > 0">
                 <span
                     v-show="!expanded"
                     class="expand expand-sidebar noselect"
@@ -55,7 +67,7 @@
 
             <div
                 class="header"
-                :class="{ hideable: sites.length > 0 }"
+                :class="{ hideable: sites.length > 0 || networks.length > 0 }"
             >
                 <a
                     v-tooltip.bottom="`Go to ${network.url}`"
@@ -72,7 +84,7 @@
             </div>
 
             <div class="content-inner">
-                <template v-if="sites.length > 0">
+                <template v-if="sites.length > 0 || networks.length > 0">
                     <span
                         v-show="expanded"
                         class="expand collapse-header noselect"
@@ -108,6 +120,7 @@
 import FilterBar from '../header/filter-bar.vue';
 import Releases from '../releases/releases.vue';
 import Sites from '../sites/sites.vue';
+import Network from '../tile/network.vue';
 
 async function fetchNetwork() {
     this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug);
@@ -119,6 +132,7 @@ async function fetchNetwork() {
         }));
     }
 
+    this.networks = this.network.networks;
     this.sites = this.network.sites
         .filter(site => !site.independent);
 
@@ -135,11 +149,13 @@ export default {
         FilterBar,
         Releases,
         Sites,
+        Network,
     },
     data() {
         return {
             network: null,
             sites: [],
+            networks: [],
             studios: [],
             releases: [],
             pageTitle: null,
@@ -212,6 +228,15 @@ export default {
     filter: $logo-highlight;
 }
 
+.parent {
+    display: inline-block;
+    height: 3rem;
+
+    :before {
+        content: 'Part of';
+    }
+}
+
 .header {
     width: 100%;
     display: flex;

assets/components/sites/sites.vue

@@ -38,6 +38,7 @@ export default {
 .sites {
     overflow: hidden;
     display: flex;
+    flex-grow: 1;
 
     &.compact:not(.expanded) {
         flex-direction: row;

assets/components/tile/network.vue

@@ -27,14 +27,15 @@ export default {
 @import 'theme';
 
 .tile {
+    height: 6rem;
     background: $profile;
     display: flex;
+    flex-shrink: 0;
     flex-direction: column;
     align-items: center;
     box-sizing: border-box;
     padding: .5rem 1rem;
     box-shadow: 0 0 3px rgba(0, 0, 0, .25);
-    height: 100%;
     text-align: center;
 }
 
@@ -44,7 +45,7 @@ export default {
 
 .logo {
     width: 100%;
-    height: 5rem;
+    height: 100%;
     color: $text;
     display: flex;
     align-items: center;

assets/components/tile/site.vue

@@ -27,15 +27,16 @@ export default {
 @import 'theme';
 
 .tile {
+    height: 6rem;
     background: $tile;
     display: flex;
+    flex-shrink: 0;
     flex-direction: column;
     align-items: center;
     box-sizing: border-box;
     padding: .5rem 1rem;
     border-radius: .25rem;
     box-shadow: 0 0 3px rgba(0, 0, 0, .25);
-    height: 100%;
     text-align: center;
 }
 

assets/js/curate.js

@@ -57,9 +57,12 @@ function curateNetwork(network, releases) {
         name: network.name,
         slug: network.slug,
         url: network.url,
+        networks: [],
     };
 
+    if (network.parent) curatedNetwork.parent = curateNetwork(network.parent);
     if (network.sites) curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork));
+    if (network.networks) curatedNetwork.networks = network.networks.map(subNetwork => curateNetwork(subNetwork));
     if (network.studios) curatedNetwork.studios = network.studios;
     if (releases) curatedNetwork.releases = releases.map(release => curateRelease(release));
 

assets/js/networks/actions.js

@@ -18,6 +18,14 @@ function initNetworksActions(store, _router) {
                     name
                     slug
                     url
+                    networks: childNetworks(
+                        orderBy: NAME_ASC,
+                    ) {
+                        id
+                        name
+                        slug
+                        url
+                    }
                     sites(
                         orderBy: [PRIORITY_DESC, NAME_ASC],
                         filter: {
@@ -45,12 +53,23 @@ function initNetworksActions(store, _router) {
                         slug
                         url
                     }
+                    parent {
+                        id
+                        name
+                        slug
+                        url
+                    }
                 }
                 releases(
                     first: $limit,
                     orderBy: $orderBy,
                     filter: {
-                        site: { network: { slug: { equalTo: $networkSlug } } }
+                        site: {
+                            or: [
+                                { network: { slug: { equalTo: $networkSlug } } },
+                                { network: { parent: { slug: { equalTo: $networkSlug } } } }
+                            ]
+                        }
                         date: {
                             lessThan: $before,
                             greaterThan: $after,

config/default.js

@@ -22,7 +22,7 @@ module.exports = {
             'famedigital',
         ],
         [
-            // Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable) and Wicked have their own assets
+            // Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable), Burning Angel and Wicked have their own assets
             'xempire',
             'blowpass',
         ],
@@ -40,6 +40,7 @@ module.exports = {
             'transangels',
         ],
         'wicked',
+        'burningangel',
         'brazzers',
         'milehighmedia',
         [
@@ -93,7 +94,7 @@ module.exports = {
         'freeonesLegacy',
     ],
     fetchAfter: [1, 'week'],
-    nullDateLimit: 10,
+    nullDateLimit: 3,
     media: {
         path: './media',
         thumbnailSize: 320, // width for 16:9 will be exactly 576px

migrations/20190325001339_releases.js

@@ -108,6 +108,10 @@ exports.up = knex => Promise.resolve()
         table.text('description');
         table.json('parameters');
 
+        table.integer('parent_id', 12)
+            .references('id')
+            .inTable('networks');
+
         table.string('slug', 32)
             .unique();
 

package.json

@@ -35,12 +35,12 @@
     "author": "Niels Simenon",
     "license": "ISC",
     "devDependencies": {
-        "@babel/cli": "^7.7.5",
+        "@babel/cli": "^7.8.4",
-        "@babel/core": "^7.7.5",
+        "@babel/core": "^7.8.4",
-        "@babel/plugin-proposal-optional-chaining": "^7.7.5",
+        "@babel/plugin-proposal-optional-chaining": "^7.8.3",
-        "@babel/preset-env": "^7.7.6",
+        "@babel/preset-env": "^7.8.4",
-        "@babel/register": "^7.7.4",
+        "@babel/register": "^7.8.3",
-        "autoprefixer": "^9.7.3",
+        "autoprefixer": "^9.7.4",
         "babel-eslint": "^10.0.3",
         "babel-loader": "^8.0.6",
         "babel-preset-airbnb": "^3.3.2",
@@ -49,50 +49,51 @@
         "eslint-config-airbnb": "^17.1.1",
         "eslint-config-airbnb-base": "^13.2.0",
         "eslint-loader": "^2.2.1",
-        "eslint-plugin-import": "^2.19.1",
+        "eslint-plugin-import": "^2.20.1",
         "eslint-plugin-jsx-a11y": "^6.2.3",
-        "eslint-plugin-react": "^7.17.0",
+        "eslint-plugin-react": "^7.18.3",
-        "eslint-plugin-vue": "^6.0.1",
+        "eslint-plugin-vue": "^6.2.1",
         "eslint-watch": "^4.0.2",
         "mini-css-extract-plugin": "^0.7.0",
-        "node-sass": "^4.13.0",
+        "node-sass": "^4.13.1",
         "postcss-loader": "^3.0.0",
         "raw-loader": "^2.0.0",
         "sass-loader": "^7.3.1",
         "style-loader": "^0.23.1",
-        "vue-loader": "^15.7.2",
+        "vue-loader": "^15.9.0",
-        "vue-template-compiler": "^2.6.10",
+        "vue-template-compiler": "^2.6.11",
-        "webpack": "^4.41.2",
+        "webpack": "^4.41.6",
-        "webpack-cli": "^3.3.10"
+        "webpack-cli": "^3.3.11"
     },
     "dependencies": {
         "@graphile-contrib/pg-order-by-related": "^1.0.0-beta.6",
         "@graphile-contrib/pg-simplify-inflector": "^5.0.0-beta.1",
-        "@tensorflow/tfjs-node": "^1.5.1",
+        "@tensorflow/tfjs-node": "^1.5.2",
         "babel-polyfill": "^6.26.0",
-        "bhttp": "^1.2.4",
+        "bhttp": "^1.2.6",
         "blake2": "^4.0.0",
         "bluebird": "^3.7.2",
         "body-parser": "^1.19.0",
         "canvas": "^2.6.1",
         "cheerio": "^1.0.0-rc.3",
         "cli-confirm": "^1.0.1",
-        "config": "^3.2.4",
+        "config": "^3.2.5",
-        "dayjs": "^1.8.17",
+        "dayjs": "^1.8.20",
         "express": "^4.17.1",
         "express-promise-router": "^3.0.3",
         "express-react-views": "^0.11.0",
         "face-api.js": "^0.21.0",
         "fs-extra": "^7.0.1",
         "graphile-utils": "^4.5.6",
-        "iconv-lite": "^0.5.0",
+        "iconv-lite": "^0.5.1",
         "jsdom": "^15.2.1",
         "knex": "^0.16.5",
         "knex-migrate": "^1.7.4",
+        "longjohn": "^0.2.12",
         "mime": "^2.4.4",
         "moment": "^2.24.0",
         "opn": "^5.5.0",
-        "pg": "^7.14.0",
+        "pg": "^7.18.1",
         "postgraphile": "^4.5.5",
         "postgraphile-plugin-connection-filter": "^1.1.3",
         "prop-types": "^15.7.2",
@@ -103,14 +104,14 @@
         "source-map-support": "^0.5.16",
         "template-format": "^1.2.5",
         "tough-cookie": "^3.0.1",
-        "tty-table": "^2.8.3",
+        "tty-table": "^2.8.12",
         "url-pattern": "^1.0.3",
-        "v-tooltip": "^2.0.2",
+        "v-tooltip": "^2.0.3",
-        "vue": "^2.6.10",
+        "vue": "^2.6.11",
-        "vue-router": "^3.1.3",
+        "vue-router": "^3.1.5",
         "vuex": "^3.1.2",
         "winston": "^3.2.1",
-        "winston-daily-rotate-file": "^4.4.1",
+        "winston-daily-rotate-file": "^4.4.2",
         "yargs": "^13.3.0"
     }
 }

seeds/01_networks.js

@@ -1,30 +1,48 @@
 /* eslint-disable max-len */
 const upsert = require('../src/utils/upsert');
 
+const parentNetworks = [
+    {
+        slug: 'gamma',
+        name: 'Gamma Entertainment',
+        url: 'https://www.gammaentertainment.com',
+    },
+    {
+        slug: 'mindgeek',
+        name: 'Mind Geek',
+        url: 'https://www.mindgeek.com',
+        description: '',
+    },
+];
+
 const networks = [
     {
         slug: '21sextury',
         name: '21Sextury',
         url: 'https://www.21sextury.com',
         description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
+        parent: 'gamma',
     },
     {
         slug: '21sextreme',
         name: '21Sextreme',
         url: 'https://www.21sextreme.com',
         description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars',
+        parent: 'gamma',
     },
     {
         slug: '21naturals',
         name: '21Naturals',
         url: 'https://www.21naturals.com',
         description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits',
+        parent: 'gamma',
     },
     {
         slug: 'adulttime',
         name: 'Adult Time',
         url: 'https://www.adulttime.com',
         description: 'Adult Time is a premium streaming service for adults! Watch adult movies, series, and channels from the top names in the industry.',
+        parent: 'gamma',
     },
     {
         slug: 'assylum',
@@ -36,6 +54,7 @@ const networks = [
         slug: 'babes',
         name: 'Babes',
         url: 'https://www.babes.com',
+        parent: 'mindgeek',
     },
     {
         slug: 'bang',
@@ -53,6 +72,7 @@ const networks = [
         name: 'Blowpass',
         url: 'https://www.blowpass.com',
         description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
+        parent: 'gamma',
     },
     {
         slug: 'brazzers',
@@ -60,6 +80,13 @@ const networks = [
         url: 'https://www.brazzers.com',
         description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
     },
+    {
+        slug: 'burningangel',
+        name: 'Burning Angel',
+        url: 'https://www.burningangel.com',
+        description: 'BurningAngel.com features tattoo porn with punk girls, goths, emo and the best scene girl porn online! View Joanna Angel and all of her alt pornstars in rough sex videos and hardcore porn',
+        parent: 'gamma',
+    },
     {
         slug: 'cherrypimps',
         name: 'Cherry Pimps',
@@ -77,6 +104,7 @@ const networks = [
         name: 'Digital Playground',
         url: 'https://www.digitalplayground.com',
         description: 'DigitalPlayground.com is the leader in high quality adult blockbuster movies and award winning sex parodies that feature the most exclusive pornstars online! Adult Film Database of adult movies.',
+        parent: 'mindgeek',
     },
     {
         slug: 'dogfartnetwork',
@@ -89,23 +117,27 @@ const networks = [
         name: 'Evil Angel',
         url: 'https://evilangel.com',
         description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
+        parent: 'gamma',
     },
     {
         slug: 'fantasymassage',
         name: 'Fantasy Massage',
         url: 'https://www.fantasymassage.com',
+        parent: 'gamma',
     },
     {
         slug: 'famedigital',
         name: 'Fame Digital',
         url: 'https://www.famedigital.com',
         description: 'Watch and download thousands of the best porn videos at FameDigital.com, the largest porn network on the web! The hottest teens, MILFs and more pornstars are all here!',
+        parent: 'gamma',
     },
     {
         slug: 'fakehub',
         name: 'Fake Hub',
         url: 'https://www.fakehub.com',
         description: 'Wherever they go, there is porn. Hospital, Taxis, Casting… Maybe fucking to a fake cop, fake agent or fake taxi driver. And we record it all.',
+        parent: 'mindgeek',
     },
     {
         slug: 'fullpornnetwork',
@@ -118,6 +150,7 @@ const networks = [
         name: 'Girlsway',
         url: 'https://www.girlsway.com',
         description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!',
+        parent: 'gamma',
     },
     {
         slug: 'insex',
@@ -129,6 +162,7 @@ const networks = [
         slug: 'jayrock',
         name: 'JayRock Productions',
         url: 'http://jayrockcontent.com',
+        parent: 'gamma',
     },
     {
         slug: 'julesjordan',
@@ -158,12 +192,14 @@ const networks = [
         name: 'Men',
         url: 'https://www.men.com',
         description: 'Check out the best gay porn site on the net with daily updates, award-winning original series, exclusive Men.com models and over 800 of the hottest guys in gay porn.',
+        parent: 'mindgeek',
     },
     {
         slug: 'metrohd',
         name: 'Metro HD',
         url: 'https://www.metrohd.com',
         description: 'Checkout MetroHD official pornsite featuring top rated pornstars and XXX videos.',
+        parent: 'mindgeek',
     },
     {
         slug: 'mikeadriano',
@@ -176,18 +212,14 @@ const networks = [
         name: 'Mile High Media',
         url: 'https://www.milehighmedia.com',
         description: 'MileHighMedia.com is the only niche porn network you need! Watch lesbian sex, hardcore fucking and family porn stories with the hottest teens & MILFs!',
-    },
+        parent: 'mindgeek',
-    {
-        slug: 'mindgeek',
-        name: 'Mind Geek',
-        url: 'https://www.mindgeek.com',
-        description: '',
     },
     {
         slug: 'mofos',
         name: 'MOFOS',
         url: 'https://www.mofos.com',
         description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
+        parent: 'mindgeek',
     },
     {
         slug: 'naughtyamerica',
@@ -236,12 +268,14 @@ const networks = [
         name: 'Pure Taboo',
         url: 'https://www.puretaboo.com',
         description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.',
+        parent: 'gamma',
     },
     {
         slug: 'realitykings',
         name: 'Reality Kings',
         url: 'https://www.realitykings.com',
         description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
+        parent: 'mindgeek',
     },
     {
         slug: 'score',
@@ -260,12 +294,14 @@ const networks = [
         name: 'Twistys',
         url: 'https://www.twistys.com',
         description: 'The hottest high quality glamour porn for over 18 years! Over 3700+ models and 46000+ scenes. TWISTYS.com',
+        parent: 'mindgeek',
     },
     {
         slug: 'vivid',
         name: 'Vivid',
         url: 'https://www.vivid.com',
         description: 'Home of the Kim Kardashian Sex Tape, Porn Parodies, and over 30,000 XXX Movies from The World Leader In Adult Entertainment.',
+        parent: 'gamma',
     },
     {
         slug: 'vixen',
@@ -284,14 +320,30 @@ const networks = [
         name: 'Wicked',
         url: 'https://www.wicked.com',
         description: 'Welcome to the new Wicked.com! Watch over 25 years of Wicked Pictures\' brand of award-winning porn for couples and women in 4k HD movies & xxx videos',
+        parent: 'gamma',
     },
     {
         slug: 'xempire',
         name: 'XEmpire',
         url: 'https://www.xempire.com',
         description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
+        parent: 'gamma',
     },
 ];
 
 exports.seed = knex => Promise.resolve()
-    .then(async () => upsert('networks', networks, 'slug', knex));
+    .then(async () => {
+        const { inserted, updated } = await upsert('networks', parentNetworks, 'slug', knex);
+        const parentNetworksBySlug = [].concat(inserted, updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
+
+        const networksWithParent = networks.map(network => ({
+            slug: network.slug,
+            name: network.name,
+            url: network.url,
+            description: network.description,
+            parameters: network.parameters,
+            parent_id: parentNetworksBySlug[network.parent] || null,
+        }));
+
+        return upsert('networks', networksWithParent, 'slug', knex);
+    });

seeds/02_sites.js

@@ -75,7 +75,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'dominatedgirls',
@@ -85,7 +85,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'homepornreality',
@@ -95,7 +95,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'peeandblow',
@@ -105,7 +105,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'cummingmatures',
@@ -115,7 +115,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'mandyiskinky',
@@ -125,7 +125,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'speculumplays',
@@ -135,7 +135,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     {
         slug: 'creampiereality',
@@ -145,7 +145,7 @@ const sites = [
         parameters: {
             scene: 'https://21sextreme.com/en/video',
         },
-        scrape: false,
+        scrape: false, // no longer updated
     },
     // 21SEXTURY
     {
@@ -1265,6 +1265,14 @@ const sites = [
         description: "Giant black dicks paired with round asses and garnished with the tightest pussies of all colors. Butts and Blacks delivers on its name sake, only the biggest dicks rocking the thickest chicks. These round honeys can take it all in and bounce around like it's a pogo stick. Come check out these soft round asses getting the attention they deserve.",
         network: 'brazzers',
     },
+    // BURNING ANGEL
+    {
+        name: 'Burning Angel',
+        slug: 'burningangel',
+        url: 'https://www.burningangel.com',
+        network: 'burningangel',
+        parameters: { independent: true },
+    },
     // CHERRY PIMPS
     {
         slug: 'cherrypimps',

src/app.js

@@ -8,6 +8,10 @@ const scrapeSites = require('./scrape-sites');
 const { scrapeReleases } = require('./scrape-releases');
 const { scrapeActors, scrapeBasicActors } = require('./actors');
 
+if (process.env.NODE_ENV === 'development') {
+    require('longjohn'); // eslint-disable-line global-require
+}
+
 async function init() {
     if (argv.scene) {
         await scrapeReleases(argv.scene, null, 'scene');

src/media.js

@@ -43,7 +43,7 @@ async function createThumbnail(buffer) {
 function pluckItems(items, specifiedLimit) {
     const limit = specifiedLimit || config.media.limit;
 
-    if (items.length <= limit) return items;
+    if (!items || items.length <= limit) return items;
 
     const plucked = [1]
         .concat(
@@ -73,7 +73,9 @@ async function getEntropy(buffer) {
 }
 
 async function extractItem(source) {
-    const res = await bhttp.get(source.src);
+    const res = await bhttp.get(source.src, {
+        responseTimeout: 10000,
+    });
 
     if (res.statusCode === 200) {
         const { q } = ex(res.body.toString());
@@ -114,7 +116,9 @@ async function fetchItem(source, index, existingItemsBySource, domain, role, att
 
         logger.verbose(`Fetching media item from ${source.src || source}`);
 
-        const res = await bhttp.get(source.src || source);
+        const res = await bhttp.get(source.src || source, {
+            responseTimeout: 10000,
+        });
 
         if (res.statusCode === 200) {
             const { pathname } = new URL(source.src || source);
@@ -199,6 +203,8 @@ async function saveItems(items, domain, role) {
             logger.error(`Failed to store ${domain} ${role} from ${item.source}: ${error.message}`);
             return null;
         }
+    }, {
+        concurrency: 20,
     });
 }
 

src/releases.js

@@ -291,6 +291,8 @@ function accumulateActors(releases) {
             const actorName = actor.name ? actor.name.trim() : actor.trim();
             const actorSlug = slugify(actorName);
 
+            if (!actorSlug) return;
+
             if (!acc[actorSlug]) {
                 acc[actorSlug] = {
                     name: actorName,
@@ -348,8 +350,8 @@ async function storeReleaseAssets(releases) {
         associateMedia(releaseCoversById, covers, 'release', 'cover'),
     ]);
 
-    const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
+    // const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
-    await associateMedia(releasePhotosById, photos, 'release', 'photo');
+    // await associateMedia(releasePhotosById, photos, 'release', 'photo');
 
     // videos take a long time, fetch last
     const [trailers, teasers] = await Promise.all([

src/scrape-sites.js

@@ -44,6 +44,11 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
 
     const latestReleases = await scraper.fetchLatest(site, page, preflight);
 
+    if (!Array.isArray(latestReleases)) {
+        logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`);
+        return accReleases;
+    }
+
     if (latestReleases.length === 0) {
         return accReleases;
     }
@@ -55,7 +60,7 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
         .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
             && (argv.last || !release.date || moment(release.date).isAfter(afterDate))); // release is older than specified date limit
 
-    logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
+    logger.verbose(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
 
     const uniqueReleasesWithSite = uniqueReleases.map(release => ({ ...release, site }));
 

src/scrapers/burningangel.js

@@ -0,0 +1,10 @@
+'use strict';
+
+const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
+
+module.exports = {
+    fetchLatest: fetchApiLatest,
+    fetchProfile: fetchApiProfile,
+    fetchScene,
+    fetchUpcoming: fetchApiUpcoming,
+};

src/scrapers/ddfnetwork.js

@@ -131,10 +131,13 @@ async function fetchLatest(site, page = 1) {
         ? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
         : `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
 
-    console.log(url);
     const res = await bhttp.get(url);
 
-    return scrapeAll(res.body.toString(), site);
+    if (res.statusCode === 200) {
+        return scrapeAll(res.body.toString(), site);
+    }
+
+    return res.statusCode;
 }
 
 async function fetchScene(url, site) {

src/scrapers/julesjordan.js

@@ -186,7 +186,7 @@ function scrapeUpcoming(html, site) {
 
         const videoClass = $(element).find('.update_thumbnail div').attr('class');
         const videoScript = $(element).find(`script:contains(${videoClass})`).html();
-        const trailer = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
+        const teaser = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
 
         return {
             url: null,
@@ -195,8 +195,8 @@ function scrapeUpcoming(html, site) {
             date,
             actors,
             poster,
-            trailer: {
+            teaser: {
-                src: trailer,
+                src: teaser,
             },
             rating: null,
             site,

src/scrapers/score.js

@@ -202,7 +202,7 @@ async function fetchLatest(site, page = 1) {
         return scrapeAll(res.body.toString(), site);
     }
 
-    return null;
+    return res.statusCode;
 }
 
 async function fetchScene(url, site) {

src/scrapers/scrapers.js

@@ -8,6 +8,7 @@ const bangbros = require('./bangbros');
 const blowpass = require('./blowpass');
 const boobpedia = require('./boobpedia');
 const brazzers = require('./brazzers');
+const burningangel = require('./burningangel');
 const cherrypimps = require('./cherrypimps');
 const ddfnetwork = require('./ddfnetwork');
 const digitalplayground = require('./digitalplayground');
@@ -67,6 +68,7 @@ module.exports = {
         bangbros,
         blowpass,
         brazzers,
+        burningangel,
         cherrypimps,
         ddfnetwork,
         digitalplayground,
@@ -117,6 +119,7 @@ module.exports = {
         boobpedia,
         brattysis: nubiles,
         brazzers,
+        burningangel,
         cherrypimps,
         ddfnetwork,
         deeplush: nubiles,

src/sites.js

@@ -132,6 +132,7 @@ async function fetchSitesFromConfig() {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .leftJoin('networks', 'sites.network_id', 'networks.id')
+        .where('sites.scrape', true)
         .where((builder) => {
             if (config.include) {
                 builder

src/utils/slugify.js

@@ -7,6 +7,8 @@ function slugify(string, {
 } = {}) {
     const slugComponents = string.trim().toLowerCase().match(/\w+/g);
 
+    if (!slugComponents) return '';
+
     const slug = slugComponents.reduce((acc, component, index) => {
         const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`;