Added support for release photo fallbacks. Limited photo fetching concurrency. Modifier XEmpire scraper for AllBlackX support and improved photo scraping. Added movie property to Evil Angel scraper.

src/media.js

@@ -10,7 +10,6 @@ const sharp = require('sharp');
 const blake2 = require('blake2');
 
 const knex = require('./knex');
-const pluckPhotos = require('./utils/pluck-photos');
 
 function getHash(buffer) {
     const hash = blake2.createHash('blake2b', { digestLength: 24 });
@@ -20,6 +19,21 @@ function getHash(buffer) {
     return hash.digest('hex');
 }
 
+function pluckPhotos(photos, release, specifiedLimit) {
+    const limit = specifiedLimit || config.media.limit;
+
+    if (photos.length <= limit) {
+        return photos;
+    }
+
+    const plucked = [1]
+        .concat(
+            Array.from({ length: limit }, (value, index) => Math.round((index + 1) * (photos.length / (limit)))),
+        );
+
+    return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
+}
+
 async function getThumbnail(buffer) {
     return sharp(buffer)
         .resize({
@@ -94,7 +108,12 @@ async function filterHashDuplicates(files, domains = ['releases'], roles = ['pho
     return files.filter(file => file && !photoHashes.has(file.hash));
 }
 
-async function fetchPhoto(photoUrl, index, identifier) {
+async function fetchPhoto(photoUrl, index, identifier, attempt = 1) {
+    if (Array.isArray(photoUrl)) {
+        return fetchPhoto(photoUrl[0], index, identifier);
+        // return photoUrl.reduce(async (outcome, url) => outcome.catch(async () => fetchPhoto(url, index, identifier)), Promise.reject());
+    }
+
     try {
         const { pathname } = new URL(photoUrl);
         const mimetype = mime.getType(pathname);
@@ -116,7 +135,12 @@ async function fetchPhoto(photoUrl, index, identifier) {
 
         throw new Error(`Response ${res.statusCode} not OK`);
     } catch (error) {
-        console.warn(`Failed to store photo ${index + 1} (${photoUrl}) for ${identifier}: ${error}`);
+        console.warn(`Failed attempt ${attempt}/3 to fetch photo ${index + 1} (${photoUrl}) for ${identifier}: ${error}`);
+
+        if (attempt < 3) {
+            await Promise.delay(1000);
+            return fetchPhoto(photoUrl, index, identifier, attempt + 1);
+        }
 
         return null;
     }