Added generic Gamma photo and actor scraper for XEmpire, 21Sextury, Blowpass and Evil Angel.

2020-01-22 22:25:58 +01:00
parent 4e4323704a
commit f8175f6054
17 changed files with 347 additions and 290 deletions
--- a/src/media.js
+++ b/src/media.js
@@ -175,7 +175,7 @@ async function storePhotos(photos, {
        return;
    }

-    const pluckedPhotos = pluckPhotos(photos);
+    const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
    const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);

    const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {