Added 2 minute timeout to media fetch.

src/media.js

@@ -365,7 +365,8 @@ async function fetchSource(source, baseMedia) {
             }
 
             // pipeline destroys streams
-            const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
+            // const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
+            const infoPromise = once(metaStream, 'info');
             const metaPromise = type === 'image' ? metaStream.stats() : Promise.resolve();
 
             await pipeline(
@@ -375,21 +376,7 @@ async function fetchSource(source, baseMedia) {
                 tempFileTarget,
             );
 
-            /*
+            const [stats, info] = await Promise.all([metaPromise, infoPromise]);
-            res.originalRes
-                .pipe(metaStream)
-                .pipe(hashStream)
-                .pipe(tempFileTarget);
-            */
-
-            logger.silly(`Temporarily saved media from ${source.src}`);
-
-            const [stats, info] = await Promise.all([
-                metaPromise,
-                infoPromise,
-            ]);
-
-            logger.silly(`Ended pipeline for ${source.src}`);
 
             hasher.end();
 
@@ -398,7 +385,7 @@ async function fetchSource(source, baseMedia) {
 
             peakMemoryUsage = Math.max(getMemoryUsage(), peakMemoryUsage);
 
-            logger.silly(`Retrieved metadata from ${source.src}`);
+            logger.silly(`Fetched media from ${source.src}`);
 
             return {
                 ...source,
@@ -422,14 +409,21 @@ async function fetchSource(source, baseMedia) {
 
             if (attempts < 3) {
                 await Promise.delay(1000);
-                return attempt(attempts + 1);
+
+                return Promise.race([
+                    attempt(attempts + 1),
+                    Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch attempt ${attempts}/3 timed out, aborting ${source.src}`); }),
+                ]);
             }
 
             throw new Error(`Failed to fetch ${source.src}: ${error.message}`);
         }
     }
 
-    return attempt(1);
+    return Promise.race([
+        attempt(1),
+        Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch timed out, aborting ${source.src}`); }),
+    ]);
 }
 
 async function trySource(baseSource, existingMedias, baseMedia) {
@@ -461,7 +455,13 @@ async function fetchMedia(baseMedia, existingMedias) {
     try {
         const source = await baseMedia.sources.reduce(
             // try each source until success
-            (result, baseSource, baseSourceIndex) => result.catch(async () => trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)),
+            (result, baseSource, baseSourceIndex) => result.catch(async (error) => {
+                if (error.message) {
+                    logger.warn(error.message);
+                }
+
+                return trySource(baseSource, existingMedias, baseMedia, baseSourceIndex);
+            }),
             Promise.reject(new Error()),
         );
 
@@ -599,7 +599,9 @@ async function associateReleaseMedia(releases) {
             }, [])
             .filter(Boolean);
 
+        if (associations.length > 0) {
             await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
+        }
     }, Promise.resolve());
 
     logger.debug(`Peak media fetching memory usage: ${peakMemoryUsage.toFixed(2)} MB`);

src/scrapers/vixen.js

@@ -42,6 +42,32 @@ function getAvatarFallbacks(avatar) {
         .flat();
 }
 
+async function getTrailer(scene, site, url) {
+    const qualities = [360, 480, 720, 1080, 2160];
+
+    const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
+        file: scene.previewVideoUrl1080P,
+        sizes: qualities.join('+'),
+        type: 'trailer',
+    }, { referer: url });
+
+    if (!tokenRes.ok) {
+        return null;
+    }
+
+    const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
+    const trailersRes = await post(trailerUrl, null, { referer: url });
+
+    if (trailersRes.ok) {
+        return qualities.map(quality => (trailersRes.body[quality] ? {
+            src: trailersRes.body[quality].token,
+            quality,
+        } : null)).filter(Boolean);
+    }
+
+    return null;
+}
+
 function scrapeAll(scenes, site, origin) {
     return scenes.map((scene) => {
         const release = {};
@@ -90,32 +116,6 @@ function scrapeUpcoming(scene, site) {
     return [release];
 }
 
-async function getTrailer(scene, site, url) {
-    const qualities = [360, 480, 720, 1080, 2160];
-
-    const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
-        file: scene.previewVideoUrl1080P,
-        sizes: qualities.join('+'),
-        type: 'trailer',
-    }, { referer: url });
-
-    if (!tokenRes.ok) {
-        return null;
-    }
-
-    const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
-    const trailersRes = await post(trailerUrl, null, { referer: url });
-
-    if (trailersRes.ok) {
-        return qualities.map(quality => (trailersRes.body[quality] ? {
-            src: trailersRes.body[quality].token,
-            quality,
-        } : null)).filter(Boolean);
-    }
-
-    return null;
-}
-
 async function scrapeScene(data, url, site, baseRelease) {
     const scene = data.video;