Added parent-child relations to network, showing parent network in sidebar. Added Burning Angel using Gamma API.

src/app.js

@@ -8,6 +8,10 @@ const scrapeSites = require('./scrape-sites');
 const { scrapeReleases } = require('./scrape-releases');
 const { scrapeActors, scrapeBasicActors } = require('./actors');
 
+if (process.env.NODE_ENV === 'development') {
+    require('longjohn'); // eslint-disable-line global-require
+}
+
 async function init() {
     if (argv.scene) {
         await scrapeReleases(argv.scene, null, 'scene');

src/media.js

@@ -43,7 +43,7 @@ async function createThumbnail(buffer) {
 function pluckItems(items, specifiedLimit) {
     const limit = specifiedLimit || config.media.limit;
 
-    if (items.length <= limit) return items;
+    if (!items || items.length <= limit) return items;
 
     const plucked = [1]
         .concat(
@@ -73,7 +73,9 @@ async function getEntropy(buffer) {
 }
 
 async function extractItem(source) {
-    const res = await bhttp.get(source.src);
+    const res = await bhttp.get(source.src, {
+        responseTimeout: 10000,
+    });
 
     if (res.statusCode === 200) {
         const { q } = ex(res.body.toString());
@@ -114,7 +116,9 @@ async function fetchItem(source, index, existingItemsBySource, domain, role, att
 
         logger.verbose(`Fetching media item from ${source.src || source}`);
 
-        const res = await bhttp.get(source.src || source);
+        const res = await bhttp.get(source.src || source, {
+            responseTimeout: 10000,
+        });
 
         if (res.statusCode === 200) {
             const { pathname } = new URL(source.src || source);
@@ -199,6 +203,8 @@ async function saveItems(items, domain, role) {
             logger.error(`Failed to store ${domain} ${role} from ${item.source}: ${error.message}`);
             return null;
         }
+    }, {
+        concurrency: 20,
     });
 }
 

src/releases.js

@@ -291,6 +291,8 @@ function accumulateActors(releases) {
             const actorName = actor.name ? actor.name.trim() : actor.trim();
             const actorSlug = slugify(actorName);
 
+            if (!actorSlug) return;
+
             if (!acc[actorSlug]) {
                 acc[actorSlug] = {
                     name: actorName,
@@ -348,8 +350,8 @@ async function storeReleaseAssets(releases) {
         associateMedia(releaseCoversById, covers, 'release', 'cover'),
     ]);
 
-    const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
-    await associateMedia(releasePhotosById, photos, 'release', 'photo');
+    // const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
+    // await associateMedia(releasePhotosById, photos, 'release', 'photo');
 
     // videos take a long time, fetch last
     const [trailers, teasers] = await Promise.all([

src/scrape-sites.js

@@ -44,6 +44,11 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
 
     const latestReleases = await scraper.fetchLatest(site, page, preflight);
 
+    if (!Array.isArray(latestReleases)) {
+        logger.warn(`Scraper returned ${latestReleases || 'null'} when fetching latest from '${site.name}' on '${site.network.name}'`);
+        return accReleases;
+    }
+
     if (latestReleases.length === 0) {
         return accReleases;
     }
@@ -55,7 +60,7 @@ async function scrapeUniqueReleases(scraper, site, preflight, afterDate = getAft
         .filter(release => !duplicateReleaseIds.has(String(release.entryId)) // release is already in database
             && (argv.last || !release.date || moment(release.date).isAfter(afterDate))); // release is older than specified date limit
 
-    logger.info(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
+    logger.verbose(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique recent releases`);
 
     const uniqueReleasesWithSite = uniqueReleases.map(release => ({ ...release, site }));
 

src/scrapers/burningangel.js

@@ -0,0 +1,10 @@
+'use strict';
+
+const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
+
+module.exports = {
+    fetchLatest: fetchApiLatest,
+    fetchProfile: fetchApiProfile,
+    fetchScene,
+    fetchUpcoming: fetchApiUpcoming,
+};

src/scrapers/ddfnetwork.js

@@ -131,10 +131,13 @@ async function fetchLatest(site, page = 1) {
         ? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
         : `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
 
-    console.log(url);
     const res = await bhttp.get(url);
 
-    return scrapeAll(res.body.toString(), site);
+    if (res.statusCode === 200) {
+        return scrapeAll(res.body.toString(), site);
+    }
+
+    return res.statusCode;
 }
 
 async function fetchScene(url, site) {

src/scrapers/julesjordan.js

@@ -186,7 +186,7 @@ function scrapeUpcoming(html, site) {
 
         const videoClass = $(element).find('.update_thumbnail div').attr('class');
         const videoScript = $(element).find(`script:contains(${videoClass})`).html();
-        const trailer = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
+        const teaser = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4);
 
         return {
             url: null,
@@ -195,8 +195,8 @@ function scrapeUpcoming(html, site) {
             date,
             actors,
             poster,
-            trailer: {
-                src: trailer,
+            teaser: {
+                src: teaser,
             },
             rating: null,
             site,

src/scrapers/score.js

@@ -202,7 +202,7 @@ async function fetchLatest(site, page = 1) {
         return scrapeAll(res.body.toString(), site);
     }
 
-    return null;
+    return res.statusCode;
 }
 
 async function fetchScene(url, site) {

src/scrapers/scrapers.js

@@ -8,6 +8,7 @@ const bangbros = require('./bangbros');
 const blowpass = require('./blowpass');
 const boobpedia = require('./boobpedia');
 const brazzers = require('./brazzers');
+const burningangel = require('./burningangel');
 const cherrypimps = require('./cherrypimps');
 const ddfnetwork = require('./ddfnetwork');
 const digitalplayground = require('./digitalplayground');
@@ -67,6 +68,7 @@ module.exports = {
         bangbros,
         blowpass,
         brazzers,
+        burningangel,
         cherrypimps,
         ddfnetwork,
         digitalplayground,
@@ -117,6 +119,7 @@ module.exports = {
         boobpedia,
         brattysis: nubiles,
         brazzers,
+        burningangel,
         cherrypimps,
         ddfnetwork,
         deeplush: nubiles,

src/sites.js

@@ -132,6 +132,7 @@ async function fetchSitesFromConfig() {
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
         .leftJoin('networks', 'sites.network_id', 'networks.id')
+        .where('sites.scrape', true)
         .where((builder) => {
             if (config.include) {
                 builder

src/utils/slugify.js

@@ -7,6 +7,8 @@ function slugify(string, {
 } = {}) {
     const slugComponents = string.trim().toLowerCase().match(/\w+/g);
 
+    if (!slugComponents) return '';
+
     const slug = slugComponents.reduce((acc, component, index) => {
         const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`;