Inspecting performance.

src/actors.js

@@ -442,10 +442,12 @@ async function associateActors(mappedActors, releases) {
         }
     });
 
-    await Promise.all([
-        knex('releases_actors').insert(associations.filter(association => association).flat()),
-        scrapeBasicActors(),
-    ]);
+    await knex('releases_actors').insert(associations.filter(association => association).flat());
+
+    // basic actor scraping is failure prone, don't together with actor association
+    // await scrapebasicactors(),
+
+    return;
 }
 
 module.exports = {

src/media.js

@@ -37,7 +37,8 @@ function pluckPhotos(photos, specifiedLimit) {
 }
 
 async function createThumbnail(buffer) {
-    return sharp(buffer)
+    try {
+        const thumbnail = sharp(buffer)
             .resize({
                 height: config.media.thumbnailSize,
                 withoutEnlargement: true,
@@ -46,6 +47,12 @@ async function createThumbnail(buffer) {
                 quality: config.media.thumbnailQuality,
             })
             .toBuffer();
+
+        return thumbnail;
+    } catch (error) {
+        logger.error(`Failed to create thumbnail: ${error.message}`);
+        throw error;
+    }
 }
 
 async function createMediaDirectory(domain, subpath) {
@@ -138,6 +145,7 @@ async function savePhotos(files, {
     naming = 'index',
 }) {
     return Promise.map(files, async (file, index) => {
+        try {
             const timestamp = new Date().getTime();
             const thumbnail = await createThumbnail(file.photo);
 
@@ -159,6 +167,10 @@ async function savePhotos(files, {
                 filepath,
                 thumbpath,
             };
+        } catch (error) {
+            logger.error(`Failed to store ${domain} ${role} to ${subpath}: ${error.message}`);
+            return null;
+        }
     });
 }
 
@@ -193,7 +205,7 @@ async function storePhotos(photos, {
         naming,
     });
 
-    const curatedPhotoEntries = curatePhotoEntries(savedPhotos, domain, role, targetId);
+    const curatedPhotoEntries = curatePhotoEntries(savedPhotos.filter(Boolean), domain, role, targetId);
 
     const newPhotos = await knex('media').insert(curatedPhotoEntries).returning('*');
     const photoEntries = Array.isArray(newPhotos)

src/releases.js

@@ -8,7 +8,7 @@ const knex = require('./knex');
 const argv = require('./argv');
 const whereOr = require('./utils/where-or');
 const { associateTags } = require('./tags');
-const { associateActors } = require('./actors');
+const { associateActors, scrapeBasicActors } = require('./actors');
 const {
     createMediaDirectory,
     storePhotos,
@@ -425,6 +425,8 @@ async function storeReleases(releases) {
         storeReleaseAssets(storedReleases),
     ]);
 
+    await scrapeBasicActors(),
+
     return {
         releases: storedReleases,
         actors,

src/scrapers/dogfart.js

@@ -131,7 +131,11 @@ async function scrapeScene(html, url, site) {
 }
 
 async function fetchLatest(site, page = 1) {
+    console.time('dogfart');
+    console.log('scraping...', site.name);
     const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+    console.timeEnd('dogfart');
+    console.log('done!', site.name);
 
     return scrapeLatest(res.body.toString(), site);
 }