Flow and modularization refactor. Added duplicates option and applying limit after fetch.

curate/submissions.js

@@ -1,10 +1,19 @@
 'use strict';
 
+const config = require('config');
 const dissectLink = require('../dissectLink.js');
 
 function curateSubmissions(submissions) {
-    return submissions.map((submission, index) => {
-        return {
+    const processed = new Set();
+
+    return submissions.reduce((acc, submission, index) => {
+        if(config.fetch.ignoreDuplicates && processed.has(submission.url)) {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
+
+            return acc;
+        }
+
+        const curatedSubmission = {
             id: submission.id,
             index: index,
             title: submission.title,
@@ -16,7 +25,11 @@ function curateSubmissions(submissions) {
             subreddit: submission.subreddit.display_name,
             host: dissectLink(submission.url)
         };
-    });
+
+        processed.add(submission.url);
+
+        return acc.concat(curatedSubmission);
+    }, []);
 };
 
 module.exports = curateSubmissions;

curate/user.js

@@ -3,8 +3,6 @@
 const path = require('path');
 
 function curateUser(user) {
-    console.log(user);
-
     return {
         id: user.id,
         name: user.name,