Added include and exclude source arguments. Improved stream fetch failure handling and improved logging details.

src/curate/posts.js

@@ -3,12 +3,14 @@
 const config = require('config');
 const dissectLink = require('../dissectLink.js');
 
-function curatePosts(posts, ignore) {
+function curatePosts(posts, args) {
     const processed = new Set();
 
     return posts.reduce((acc, post, index) => {
         const host = dissectLink(post.url);
-        const ignoring = ignore ? ignore.find(prop => {
+        post.permalink = 'https://reddit.com' + post.permalink;
+
+        const ignoring = args.ignore ? args.ignore.find(prop => {
             return post[prop];
         }) : null;
 
@@ -25,6 +27,12 @@ function curatePosts(posts, ignore) {
                 return acc;
             }
 
+            if((args.include && !args.include.includes(host.label)) || (args.exclude && args.exclude.includes(host.label))) {
+                console.log('\x1b[33m%s\x1b[0m', `Ignoring source '${host.label}' from post '${post.url}' (${post.permalink})`);
+
+                return acc;
+            }
+
             processed.add(host.id);
         }
 
@@ -34,7 +42,7 @@ function curatePosts(posts, ignore) {
             title: post.title,
             text: post.selftext,
             user: post.user,
-            permalink: 'https://reddit.com' + post.permalink,
+            permalink: post.permalink,
             url: post.url,
             datetime: new Date(post.created_utc * 1000),
             subreddit: post.subreddit.display_name,