Ignoring empty interpolated metadata. Added URL query wildcard to imgur and reddit images. Using host ID rather than URL to improve duplicate detection.

src/curate/submissions.js

@@ -7,7 +7,9 @@ function curateSubmissions(submissions) {
     const processed = new Set();
 
     return submissions.reduce((acc, submission, index) => {
-        if(config.fetch.avoidDuplicates && processed.has(submission.url)) {
+        const host = dissectLink(submission.url);
+
+        if(config.fetch.avoidDuplicates && processed.has(host.id)) {
             console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
 
             return acc;
@@ -23,10 +25,10 @@ function curateSubmissions(submissions) {
             url: submission.url,
             datetime: new Date(submission.created_utc * 1000),
             subreddit: submission.subreddit.display_name,
-            host: dissectLink(submission.url)
+            host
         };
 
-        processed.add(submission.url);
+        processed.add(host.id);
 
         return acc.concat(curatedSubmission);
     }, []);

src/dissectLink.js

@@ -13,11 +13,11 @@ const hosts = [{
 }, {
     method: 'reddit',
     label: 'reddit',
-    pattern: new urlPattern('https\\://i.reddituploads.com/:id?(fit=:fit)(&h=:height)(&w=:width)(&s=:signature)')
+    pattern: new urlPattern('https\\://i.reddituploads.com/:id(?*)')
 }, {
     method: 'imgurImage',
     label: 'imgur',
-    pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
+    pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?*)')
 }, {
     method: 'imgurAlbum',
     label: 'imgur',

src/fetch/content.js

@@ -43,7 +43,11 @@ module.exports = function(posts, user) {
                         return save(filepath, item.stream);
                     }).then(() => {
                         const interpolatedMeta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
-                            acc[key] = interpolate(value, user, post, item);
+                            const interpolatedValue = interpolate(value, user, post, item);
+
+                            if(interpolatedValue) {
+                                acc[key] = interpolatedValue;
+                            }
 
                             return acc;
                         }, {});
@@ -58,7 +62,6 @@ module.exports = function(posts, user) {
             });
         }));
     }).then(() => {
-        console.log('closing...');
         return ep.close();
     }).catch(error => {
         return ep.close();