Added option to index (specified) ignored posts. Saving index through save module, and it now notifies the user that an index has been written.

src/cli.js

@@ -73,6 +73,10 @@ function getArgs() {
             describe: 'Ignore index file and force a redownload of the profile image and description',
             type: 'boolean',
         })
+        .option('index-ignored', {
+            describe: 'Add posts specified to be ignored to the index file, but don\'t download them',
+            type: 'boolean',
+        })
         .option('watch', {
             describe: 'Keep the process running and periodically check for new posts',
             type: 'boolean',

src/curate/posts.js

@@ -9,12 +9,21 @@ const hashPost = require('./hashPost.js');
 const { isAfter, isBefore, isEqual } = require('date-fns');
 
 function report(curatedPosts, indexed, user, args) {
-    const { posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount } = curatedPosts;
+    const {
+        posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored,
+    } = curatedPosts;
 
     if (indexedUpdated.length > 0) {
         console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`);
     }
 
+    if (requestedIgnored.length > 0) {
+        console.log(
+            '\x1b[33m%s\x1b[0m',
+            `Ignoring ${requestedIgnored.length} posts because their IDs are specified to be ignored for '${user.name}'`,
+        );
+    }
+
     if (tooOldCount > 0) {
         console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`);
     }
@@ -26,14 +35,14 @@ function report(curatedPosts, indexed, user, args) {
     if (beforeIndexedCount > 0) {
         console.log(
             '\x1b[33m%s\x1b[0m',
-            `Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`
+            `Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`,
         );
     }
 
     if (afterIndexedCount > 0) {
         console.log(
             '\x1b[33m%s\x1b[0m',
-            `Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`
+            `Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`,
         );
     }
 }
@@ -97,12 +106,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         const hostExcluded = args.exclude && args.exclude.includes(host.label);
 
         if (ignoreIds.has(String(host.id).toLowerCase())) {
-            console.log(
+            return { ...acc, requestedIgnored: [...acc.requestedIgnored, curatedPost] };
-                '\x1b[33m%s\x1b[0m',
-                `Ignoring content '${post.url}' because its ID is specified to be ignored (${permalink})`,
-            );
-
-            return acc;
         }
 
         if (hostIncludes || hostExcluded) {
@@ -143,6 +147,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
     const curatedPosts = user.posts.reduce((accUserPosts, post, index) => curatePost(accUserPosts, post, user, index, indexed, ignoreIds, processed, args), {
         posts: [],
         indexedUpdated: [],
+        requestedIgnored: [],
         tooOldCount: 0,
         tooRecentCount: 0,
         beforeIndexedCount: 0,
@@ -162,6 +167,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
                 profile: user.indexed.profile,
                 original: indexedOriginal,
                 updated: curatedPosts.indexedUpdated,
+                ignored: curatedPosts.requestedIgnored,
                 oldest: indexed.oldest,
                 latest: indexed.latest,
             },

src/fetch/content.js

@@ -83,7 +83,7 @@ async function fetchSaveContent(user, ep, args) {
         concurrency: config.fetch.concurrency,
     });
 
-    return writeToIndex(posts, profilePaths, user);
+    return writeToIndex(posts, profilePaths, user, args);
 }
 
 module.exports = fetchSaveContent;

src/save/writeToIndex.js

@@ -1,16 +1,17 @@
 'use strict';
 
 const config = require('config');
-const fs = require('fs-extra');
 const yaml = require('js-yaml');
 
-const interpolate = require('../interpolate.js');
+const interpolate = require('../interpolate');
+const textToStream = require('./textToStream');
+const save = require('./save');
 
-async function writeToIndex(posts, profilePaths, user) {
+async function writeToIndex(posts, profilePaths, user, args) {
-    const filename = interpolate(config.library.index.file, user, null, false);
+    const filepath = interpolate(config.library.index.file, user, null, false);
     const now = new Date();
 
-    const newAndUpdatedEntries = posts.concat(user.indexed.updated).map((post) => {
+    const newAndUpdatedEntries = posts.concat(user.indexed.updated, args.indexIgnored ? user.indexed.ignored : []).map((post) => {
         const entryPost = {
             id: post.id,
             subreddit: post.subreddit,
@@ -41,7 +42,7 @@ async function writeToIndex(posts, profilePaths, user) {
         return;
     }
 
-    return fs.writeFile(filename, yaml.safeDump(data));
+    return save(filepath, textToStream(yaml.safeDump(data)));
 }
 
 module.exports = writeToIndex;

src/sources/getIndex.js

@@ -14,7 +14,7 @@ async function getIndex(user) {
 
         return yaml.safeLoad(indexFile);
     } catch (error) {
-        console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`);
+        console.log('\x1b[33m%s\x1b[0m', `No index file found for '${user.name}' at '${indexFilePath}'`);
 
         return { profile: { image: null, description: null }, posts: [] };
     }

src/sources/getUserPosts.js

@@ -23,12 +23,12 @@ async function getUser(username, reddit) {
 
 async function getPosts(username, reddit, args) {
     try {
-        const user = await reddit.getUser(username).getSubmissions({
+        const submissions = await reddit.getUser(username).getSubmissions({
             sort: args.sort,
             limit: Infinity,
         });
 
-        return user;
+        return submissions;
     } catch (error) {
         console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);