Inspecting performance.

src/actors.js

@@ -442,10 +442,12 @@ async function associateActors(mappedActors, releases) {
         }
     });
 
-    await Promise.all([
+    await knex('releases_actors').insert(associations.filter(association => association).flat());
-        knex('releases_actors').insert(associations.filter(association => association).flat()),
+
-        scrapeBasicActors(),
+    // basic actor scraping is failure prone, don't together with actor association
-    ]);
+    // await scrapebasicactors(),
+
+    return;
 }
 
 module.exports = {

src/media.js

@@ -37,15 +37,22 @@ function pluckPhotos(photos, specifiedLimit) {
 }
 
 async function createThumbnail(buffer) {
-    return sharp(buffer)
+    try {
-        .resize({
+        const thumbnail = sharp(buffer)
-            height: config.media.thumbnailSize,
+            .resize({
-            withoutEnlargement: true,
+                height: config.media.thumbnailSize,
-        })
+                withoutEnlargement: true,
-        .jpeg({
+            })
-            quality: config.media.thumbnailQuality,
+            .jpeg({
-        })
+                quality: config.media.thumbnailQuality,
-        .toBuffer();
+            })
+            .toBuffer();
+
+        return thumbnail;
+    } catch (error) {
+        logger.error(`Failed to create thumbnail: ${error.message}`);
+        throw error;
+    }
 }
 
 async function createMediaDirectory(domain, subpath) {
@@ -138,27 +145,32 @@ async function savePhotos(files, {
     naming = 'index',
 }) {
     return Promise.map(files, async (file, index) => {
-        const timestamp = new Date().getTime();
+        try {
-        const thumbnail = await createThumbnail(file.photo);
+            const timestamp = new Date().getTime();
+            const thumbnail = await createThumbnail(file.photo);
 
-        const filename = naming === 'index'
+            const filename = naming === 'index'
-            ? `${file.role || role}${index + 1}`
+                ? `${file.role || role}${index + 1}`
-            : `${timestamp + index}`;
+                : `${timestamp + index}`;
 
-        const filepath = path.join(`${domain}s`, subpath, `${filename}.${file.extension}`);
+            const filepath = path.join(`${domain}s`, subpath, `${filename}.${file.extension}`);
-        const thumbpath = path.join(`${domain}s`, subpath, `${filename}_thumb.${file.extension}`);
+            const thumbpath = path.join(`${domain}s`, subpath, `${filename}_thumb.${file.extension}`);
 
-        await Promise.all([
+            await Promise.all([
-            fs.writeFile(path.join(config.media.path, filepath), file.photo),
+                fs.writeFile(path.join(config.media.path, filepath), file.photo),
-            fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
+                fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
-        ]);
+            ]);
 
-        return {
+            return {
-            ...file,
+                ...file,
-            thumbnail,
+                thumbnail,
-            filepath,
+                filepath,
-            thumbpath,
+                thumbpath,
-        };
+            };
+        } catch (error) {
+            logger.error(`Failed to store ${domain} ${role} to ${subpath}: ${error.message}`);
+            return null;
+        }
     });
 }
 
@@ -193,7 +205,7 @@ async function storePhotos(photos, {
         naming,
     });
 
-    const curatedPhotoEntries = curatePhotoEntries(savedPhotos, domain, role, targetId);
+    const curatedPhotoEntries = curatePhotoEntries(savedPhotos.filter(Boolean), domain, role, targetId);
 
     const newPhotos = await knex('media').insert(curatedPhotoEntries).returning('*');
     const photoEntries = Array.isArray(newPhotos)

src/releases.js

@@ -8,7 +8,7 @@ const knex = require('./knex');
 const argv = require('./argv');
 const whereOr = require('./utils/where-or');
 const { associateTags } = require('./tags');
-const { associateActors } = require('./actors');
+const { associateActors, scrapeBasicActors } = require('./actors');
 const {
     createMediaDirectory,
     storePhotos,
@@ -425,6 +425,8 @@ async function storeReleases(releases) {
         storeReleaseAssets(storedReleases),
     ]);
 
+    await scrapeBasicActors(),
+
     return {
         releases: storedReleases,
         actors,

src/scrapers/dogfart.js

@@ -131,7 +131,11 @@ async function scrapeScene(html, url, site) {
 }
 
 async function fetchLatest(site, page = 1) {
+    console.time('dogfart');
+    console.log('scraping...', site.name);
     const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+    console.timeEnd('dogfart');
+    console.log('done!', site.name);
 
     return scrapeLatest(res.body.toString(), site);
 }