Photo plucker will use discarded photos as fallback. Returning high res photo sources from LegalPorno.

src/media.js

@@ -42,11 +42,50 @@ async function createThumbnail(buffer) {
     return null;
 }
 
-function pluckItems(items, specifiedLimit) {
+function groupFallbacksByPriority(chunks) {
+    /*
+        Chunks naturally give priority to all of the first item's fallbacks, generally lower quality images.
+        This function ensures every item's first source is tried, before trying every item's second source, etc., example:
+        IN: [[1, 2, 3,], 10, [1, 2, 3, 4, 5], [1, 2, 3]]
+        OUT [[1, 1, 1], [2, 2, 2], [3, 3, 3], [4], [5]]
+    */
+    return chunks.map(group => group.reduce((acc, item) => {
+        if (Array.isArray(item)) {
+            // place provided fallbacks at same index (priority) in parent array
+            item.forEach((fallback, fallbackIndex) => {
+                if (!acc[fallbackIndex]) {
+                    acc[fallbackIndex] = [];
+                }
+
+                acc[fallbackIndex].push(fallback);
+            });
+
+            return acc;
+        }
+
+        // no fallbacks provided, first priority
+        if (!acc[0]) {
+            acc[0] = [];
+        }
+
+        acc[0].push(item);
+
+        return acc;
+    }, []).flat());
+}
+
+function pluckItems(items, specifiedLimit, asFallbacks = true) {
     const limit = specifiedLimit || config.media.limit;
 
     if (!items || items.length <= limit) return items;
 
+    if (asFallbacks) {
+        const chunks = chunk(items, Math.ceil(items.length / limit));
+        const fallbacks = groupFallbacksByPriority(chunks);
+
+        return fallbacks;
+    }
+
     const plucked = [1]
         .concat(
             Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (items.length / (limit - 1)))),
@@ -93,8 +132,8 @@ async function extractItem(source) {
     return null;
 }
 
-async function fetchSource(source, domain, role, originalSource) {
-    logger.verbose(`Fetching ${domain} ${role} from ${source.src || source}`);
+async function fetchSource(source, domain, role) {
+    logger.silly(`Fetching ${domain} ${role} from ${source.src || source}`);
 
     // const res = await bhttp.get(source.src || source);
     const res = await get(source.src || source, {
@@ -111,7 +150,7 @@ async function fetchSource(source, domain, role, originalSource) {
         const hash = getHash(res.body);
         const { entropy, size, width, height } = /image/.test(mimetype) ? await getMeta(res.body) : {};
 
-        logger.verbose(`Fetched media item from ${source.src || source}`);
+        logger.silly(`Fetched media item from ${source.src || source}`);
 
         return {
             file: res.body,
@@ -123,7 +162,7 @@ async function fetchSource(source, domain, role, originalSource) {
             width: width || null,
             height: height || null,
             quality: source.quality || null,
-            source: originalSource?.src || originalSource || source.src || source,
+            source: source.src || source,
             scraper: source.scraper,
             copyright: source.copyright,
         };
@@ -133,9 +172,11 @@ async function fetchSource(source, domain, role, originalSource) {
 }
 
 async function fetchItem(source, index, existingItemsBySource, domain, role, attempt = 1, originalSource = null, sourceIndex = 0) {
-    if (!source) return null;
-
     try {
+        if (!source) {
+            throw new Error(`Empty ${domain} ${role} source in ${originalSource}`);
+        }
+
         if (Array.isArray(source)) {
             if (source.every(sourceX => sourceX.quality)) {
                 // various video qualities provided
@@ -160,19 +201,18 @@ async function fetchItem(source, index, existingItemsBySource, domain, role, att
             return null;
         }
 
-        return fetchSource(source, domain, role, originalSource);
+        return await fetchSource(source, domain, role, originalSource);
     } catch (error) {
         logger.warn(`Failed attempt ${attempt}/3 to fetch ${domain} ${role} ${index + 1} (${source.src || source}): ${error}`);
 
-        /*
-        if (attempt < 3) {
+        if (source && attempt < 3) {
+            // only retry if source is provided at all
             await Promise.delay(5000);
             return fetchItem(source, index, existingItemsBySource, domain, role, attempt + 1, originalSource, sourceIndex);
         }
-        */
 
-        if (originalSource && sourceIndex < originalSource.length) {
-            throw error;
+        if (originalSource && sourceIndex < originalSource.length - 1) {
+            throw error; // gets caught to try next source
         }
 
         return null;
@@ -285,6 +325,8 @@ async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
         return {};
     }
 
+    console.log(presentSources, presentSources.length);
+
     // split up source list to prevent excessive RAM usage
     const itemChunksBySource = await Promise.all(chunk(presentSources, 50).map(async (sourceChunk, index) => {
         try {
@@ -354,12 +396,12 @@ function associateTargetMedia(targetId, sources, mediaBySource, domain, role, pr
         .map((source) => {
             if (!source) return null;
 
-            const mediaItem = Array.isArray(source)
-                ? mediaBySource[source.map(sourceX => sourceX.src || sourceX).toString()]
-                : mediaBySource[source.src || source];
+            if (Array.isArray(source)) {
+                const availableSource = source.find(fallbackSource => mediaBySource[fallbackSource.src || fallbackSource]);
+                return mediaBySource[availableSource];
+            }
 
-            // return mediaItem && { [`${domain}_id`]: targetId, media_id: mediaItem.id };
-            return mediaItem;
+            return mediaBySource[source.src || source];
         })
         .filter(Boolean)
         // .sort((mediaItemA, mediaItemB) => mediaItemB.height - mediaItemA.height) // prefer high res images for primary item