Saving index per user as TSV. Refactoring.

src/app.js

@@ -19,7 +19,7 @@ const curatePosts = require('./curate/posts.js');
 const interpolate = require('./interpolate.js');
 
 const attachContentInfo = require('./fetch/info.js');
-const fetchContent = require('./fetch/content.js');
+const fetchSaveContent = require('./fetch/content.js');
 
 const getPosts = require('./sources/getPosts.js')(reddit, args);
 const getUserPosts = require('./sources/getUserPosts.js')(reddit, args);
@@ -42,7 +42,7 @@ Promise.resolve().then(async () => {
     const infoUserPosts = await attachContentInfo(curatedUserPosts);
 
     await ep.open();
-    await Promise.all(Object.values(infoUserPosts).map(user => fetchContent(user, ep)));
+    await Promise.all(Object.values(infoUserPosts).map(user => fetchSaveContent(user, ep)));
     await ep.close();
 }).catch(error => {
     return console.error(error);

src/fetch/content.js

@@ -8,9 +8,10 @@ const save = require('../save/save.js');
 const textToStream = require('../save/textToStream.js');
 const saveMeta = require('../save/meta.js');
 const mux = require('../save/mux.js');
+const writeToIndex = require('../save/writeToIndex.js');
 
 async function getStreams(item, post) {
-    if(item.self) {
+    if (item.self) {
         return [textToStream(item.text)];
     }
 
@@ -24,9 +25,29 @@ async function getStreams(item, post) {
     return null;
 }
 
-async function fetchContent(user, ep) {
-    await Promise.map(user.posts, async (post) => {
-        const items = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
+async function addMeta(filepath, ep, item, post, user) {
+    const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
+        const interpolatedValue = interpolate(value, user, post, item);
+
+        return interpolatedValue ? { ...acc, [key]: interpolatedValue } : acc;
+    }, {});
+
+    if (Object.keys(meta).length > 0) {
+        await saveMeta(filepath, meta, ep);
+    }
+}
+
+function getFilepath(item, post, user) {
+    const type = item.type.split('/')[0];
+
+    return post.content.album
+        ? interpolate(config.library.album[type], user, post, item)
+        : interpolate(config.library[type], user, post, item);
+}
+
+async function fetchSaveContent(user, ep) {
+    const posts = await Promise.map(user.posts, async (post) => {
+        await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
             const item = { ...originalItem, index };
             const streams = await getStreams(item, post);
 
@@ -35,36 +56,22 @@ async function fetchContent(user, ep) {
                 return accItems;
             }
 
-            const type = item.type.split('/')[0];
-            const filepath = post.content.album
-                ? interpolate(config.library.album[type], user, post, item)
-                : interpolate(config.library[type], user, post, item);
-
+            const filepath = getFilepath(item, post, user);
             const sourcePaths = await save(filepath, streams, item, post);
 
             if (item.mux) {
                 await mux(filepath, sourcePaths, item);
             }
 
-            const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
-                const interpolatedValue = interpolate(value, user, post, item);
-
-                return interpolatedValue ? { ...acc, [key]: interpolatedValue } : acc;
-            }, {});
-
-            if (Object.keys(meta).length > 0) {
-                await saveMeta(filepath, meta, ep);
-            }
+            await addMeta(filepath, ep, item, post, user);
 
             return sourcePaths;
         }, []);
 
-        console.log(items);
-
-        const filename = interpolate(config.library.index.file, user, post);
-        const entry = `${interpolate(config.library.index.entry, user, post, null, false)}\n`;
-
-        await fs.appendFile(filename, config.library.index.unique ? `${post.hash} ${entry}` : entry);
+        return post;
     });
+
+    return writeToIndex(posts, user);
 }
-module.exports = fetchContent;
+
+module.exports = fetchSaveContent;

src/interpolate.js

@@ -6,14 +6,12 @@ const url = require('url');
 const dateFns = require('date-fns');
 const mime = require('mime-types');
 
-function interpolate(pattern, user, post, item, strip = true) {
-    const dateFormat = config.library.dateFormat || 'YYYYMMDD';
-
+function interpolate(pattern, user, post, item, strip = true, dateFormat = config.library.dateFormat) {
     const vars = {
-        $base: config.library.base
+        $base: config.library.base,
     };
 
-    if(user) {
+    if (user) {
         Object.assign(vars, {
             $user: user.name,
             $username: user.name,
@@ -24,40 +22,41 @@ function interpolate(pattern, user, post, item, strip = true) {
             $userGold: user.gold ? config.library.booleans.gold : '',
         });
 
-        if(user.profile) {
+        if (user.profile) {
             Object.assign(vars, {
                 $profileId: user.profile.id,
                 $profileTitle: user.profile.title,
                 $profileDescription: user.profile.description,
-                $profileOver18: user.profile.over18 ? config.library.booleans.over18 : ''
+                $profileOver18: user.profile.over18 ? config.library.booleans.over18 : '',
             });
         }
     }
 
-    if(post) {
+    if (post) {
         Object.assign(vars, {
             $postId: post.id,
             $postTitle: (post.title || '').slice(0, config.library.titleLength),
             $postUser: post.user,
             $postDate: dateFns.format(post.datetime, dateFormat),
             $postIndex: post.index + config.library.indexOffset,
+            $postHash: post.hash,
             $url: post.url,
             $subreddit: post.subreddit,
             $hostLabel: post.host.label,
-            $hostId: post.host.id
+            $hostId: post.host.id,
         });
 
-        if(post.content.album) {
+        if (post.content.album) {
             Object.assign(vars, {
                 $albumId: post.content.album.id,
                 $albumTitle: (post.content.album.title || '').slice(0, config.library.titleLength),
                 $albumDescription: post.content.album.description,
-                $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
+                $albumDate: dateFns.format(post.content.album.datetime, dateFormat),
             });
         }
     }
 
-    if(item) {
+    if (item) {
         Object.assign(vars, {
             $itemId: item.id,
             $itemTitle: (item.title || '').slice(0, config.library.titleLength),
@@ -66,18 +65,18 @@ function interpolate(pattern, user, post, item, strip = true) {
             $itemIndex: item.index + config.library.indexOffset,
             $extracted: item.extracted ? config.library.booleans.extracted : '',
             $preview: item.preview ? config.library.booleans.preview : '',
-            $ext: item.type ? `.${mime.extension(item.type)}` : path.extname(url.parse(item.url).pathname)
+            $ext: item.type ? `.${mime.extension(item.type)}` : path.extname(url.parse(item.url).pathname),
         });
     }
 
-    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
+    return Object.entries(vars).reduce((acc, [key, value]) => {
         // substitute slashes for filesystem compatability
-        if(key !== '$base' && strip) {
+        if (key !== '$base' && strip) {
             value = (value || '').toString().replace(/\//g, config.library.slashSubstitute);
         }
 
         return acc.replace(key, value);
     }, pattern);
-};
+}
 
 module.exports = interpolate;