Refactoring flow. Added user and profile saving and variables.

app.js

@@ -2,20 +2,53 @@
 
 const config = require('config');
 const util = require('util');
+const fs = require('fs-extra');
 const yargs = require('yargs').argv;
 const snoowrap = require('snoowrap');
+
+const curateSubmissions = require('./curate/submissions.js');
+const curateUser = require('./curate/user.js');
 const methods = require('./methods/methods.js');
-const curate = require('./curate.js');
+const interpolate = require('./interpolate.js');
+const fetchItem = require('./fetchItem.js');
 const fetchContent = require('./fetchContent.js');
+const save = require('./save.js');
+const textToStream = require('./textToStream.js');
 
 const reddit = new snoowrap(config.reddit.api);
 
-function getSubmissions(users) {
+function saveProfileDetails(user) {
+    if(config.library.profile.image) {
+        // pass profile image as item to interpolate extension variable
+        const filepath = interpolate(config.library.profile.image, user, null, {
+            url: user.profile.image
+        });
+
+        fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
+            console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
+        });
+    }
+
+    if(config.library.profile.description) {
+        if(user.profile.description) {
+            const filepath = interpolate(config.library.profile.description, user);
+            const stream = textToStream(user.profile.description);
+
+            save(filepath, stream).catch(error => {
+                console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
+            });
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
+        }
+    }
+};
+
+function getSubmissions(users, sort, limit) {
     return users.reduce((chain, user) => {
         return chain.then(acc => {
             return reddit.getUser(user).getSubmissions({
-                sort: yargs.sort || config.reddit.sort,
+                sort: sort,
-                limit: yargs.limit || config.reddit.limit
+                limit: limit
             }).then(submissions => {
                 return acc.concat(submissions);
             });
@@ -23,11 +56,36 @@ function getSubmissions(users) {
     }, Promise.resolve([]));
 };
 
+if(!yargs.user && typeof yargs.users !== 'string') {
+    return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user=[user], or multiple users with --users=[user1,user2] or --user=[user1] --user=[user2]');
+}
+
+const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
+
+users.forEach(user => {
+    return Promise.resolve().then(() => {
+        // get reddit profile
+        return reddit.getUser(user).fetch().then(curateUser);
+    }).then(user => {
+        return saveProfileDetails(user);
+        // get submissions
+    }).catch(error => {
+        return console.log('\x1b[33m%s\x1b[0m', error);
+    });
+});
+
+/*
 Promise.resolve().then(() => {
     if(yargs.user || yargs.users) {
         const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
 
-        return getSubmissions(users);
+        return Promise.resolve().then(() => {
+            if(config.library.profile) {
+                return getProfiles(users);
+            }
+        }).then(() => {
+            return getSubmissions(users, yargs.sort || config.reddit.sort, yargs.limit === undefined ? config.reddit.limit : yargs.limit);
+        });
     }
 
     return Promise.reject('Please supply at least one user with one or multiple --user, or --users!');
@@ -48,5 +106,6 @@ Promise.resolve().then(() => {
 }).then(posts => {
     return fetchContent(posts);
 }).catch(error => {
-    return console.log('\x1b[33m%s\x1b[0m', error);
+    return console.log('\x1b[31m%s\x1b[0m', error);
 });
+*/

config/default.js

@@ -1,18 +1,29 @@
 module.exports = {
-    patterns: {
+    library: {
-        image: 'output/$postUser/$postDate - $itemId - $postTitle$ext',
+        base: 'output/$user/',
-        video: 'output/$postUser/$postDate - $itemId - $postTitle$ext',
+        image: '$postDate - $itemId - $postTitle$ext',
-        text: 'output/$postUser/$postDate - $postId - $postTitle',
+        video: '$postDate - $itemId - $postTitle$ext',
+        text: '$postDate - $postId - $postTitle',
         album: {
             extractSingleItem: true,
-            image: 'output/$postUser/$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
+            image: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
-            video: 'output/$postUser/$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
+            video: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
+        },
+        profile: {
+            image: 'profile$ext',
+            description: 'profile ($userVerified$userVerifiedEmail$userGold)'
+        },
+        booleans: {
+            extracted: 'extracted-',
+            verified: '✔',
+            verifiedEmail: '✉',
+            gold: '★',
+            over18: '♥'
         },
         dateFormat: 'YYYYMMDD',
         titleLength: 200,
         indexOffset: 1,
         slashSubstitute: '#',
-        extractedLabel: 'extracted-'
     },
     reddit: {
         sort: 'top',

curate/submissions.js

@@ -1,8 +1,8 @@
 'use strict';
 
-const dissectLink = require('./dissectLink.js');
+const dissectLink = require('../dissectLink.js');
 
-function curate(submissions) {
+function curateSubmissions(submissions) {
     return submissions.map((submission, index) => {
         return {
             id: submission.id,
@@ -19,4 +19,4 @@ function curate(submissions) {
     });
 };
 
-module.exports = curate;
+module.exports = curateSubmissions;

curate/user.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const path = require('path');
+
+function curateUser(user) {
+    console.log(user);
+
+    return {
+        id: user.id,
+        name: user.name,
+        created: new Date(user.created_utc * 1000),
+        gold: user.is_gold,
+        verified: user.verified,
+        verifiedEmail: user.has_verified_email,
+        profile: {
+            id: user.subreddit.display_name.name,
+            title: user.subreddit.display_name.title,
+            image: user.subreddit.display_name.icon_img,
+            banner: user.subreddit.display_name.banner_img,
+            description: user.subreddit.display_name.public_description,
+            over18: user.subreddit.display_name.over_18
+        }
+    };
+};
+
+module.exports = curateUser;

fetchContent.js

@@ -1,58 +1,13 @@
 'use strict';
 
-const Readable = require('stream').Readable;
 const fs = require('fs-extra');
 const path = require('path');
 const config = require('config');
-const fetch = require('node-fetch');
+
+const fetchItem = require('./fetchItem');
+const save = require('./save.js');
 const interpolate = require('./interpolate.js');
-
+const textToStream = require('./textToStream.js');
-function saveItemToDisk(stream, filepath) {
-    const file = fs.createWriteStream(filepath);
-
-    return new Promise((resolve, reject) => {
-        stream.pipe(file).on('error', error => {
-            reject(error);
-        }).on('finish', () => {
-            console.log(`Saved '${filepath}'`);
-
-            resolve(filepath);
-        });
-    });
-};
-
-function textPostToStream(item) {
-    const stream = new Readable();
-
-    stream.push(item.text);
-    stream.push(null);
-
-    return Object.assign(item, {
-        stream: stream
-    });
-};
-
-function fetchItem(item, post, attempt) {
-    function retry(error) {
-        console.log(error);
-
-        if(attempt < 3) {
-            console.log('Retrying...');
-
-            return fetchItem(item, post, ++attempt);
-        }
-    };
-
-    return fetch(item.url).then(res => {
-        return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
-    }).then(res => {
-        console.log(`Fetched '${item.url}'`);
-
-        return Object.assign({}, item, {
-            stream: res.body
-        });
-    }).catch(retry);
-};
 
 module.exports = function(posts) {
     return Promise.all(posts.map(post => {
@@ -61,20 +16,22 @@ module.exports = function(posts) {
                 item.index = index;
 
                 if(item.self) {
-                    return textPostToStream(item);
+                    return Object.assign({}, item, {stream: textToStream(item.text)});
                 }
 
-                return fetchItem(item, post, 0);
+                return fetchItem(item.url, 0).then(stream => {
+                    return Object.assign({}, item, {stream});
+                });
             }));
         }).then(items => {
             return Promise.all(items.map(item => {
                 const type = item.type.split('/')[0];
-                const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item) : interpolate(config.patterns[type], post, item);
+                const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
 
                 return Promise.resolve().then(() => {
                     return fs.ensureDir(path.dirname(filepath));
                 }).then(() => {
-                    return saveItemToDisk(item.stream, filepath)
+                    return save(filepath, item.stream)
                 });
             }));
         });

fetchItem.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const fetch = require('node-fetch');
+
+function fetchItem(url, attempt) {
+    function retry(error) {
+        console.log(error);
+
+        if(attempt < 3) {
+            console.log('Retrying...');
+
+            return fetchItem(url, ++attempt);
+        }
+    };
+
+    return fetch(url).then(res => {
+        return res.ok ? res : Promise.reject(`Failed to fetch ${url}`);
+    }).then(res => {
+        console.log(`Fetched '${url}'`);
+
+        return res.body;
+    }).catch(retry);
+};
+
+module.exports = fetchItem;

interpolate.js

@@ -1,6 +1,8 @@
 'use strict';
 
 const config = require('config');
+const path = require('path');
+const url = require('url');
 const dateFns = require('date-fns');
 
 const extensions = {
@@ -10,45 +12,68 @@ const extensions = {
     'video/webm': '.webm'
 };
 
-function interpolate(path, post, item) {
+function interpolate(pattern, user, post, item) {
-    const dateFormat = config.patterns.dateFormat || 'YYYYMMDD';
+    const dateFormat = config.library.dateFormat || 'YYYYMMDD';
+    const vars = {};
 
-    const vars = {
+    if(config.library.base) {
+        pattern = path.join(config.library.base, pattern);
+    }
+
+    if(user) {
+        Object.assign(vars, {
+            $user: user.name,
+            $username: user.name,
+            $userId: user.id,
+            $userCreated: dateFns.format(user.created, dateFormat),
+            $userVerified: user.verified ? config.library.booleans.verified : '',
+            $userVerifiedEmail: user.verifiedEmail ? config.library.booleans.verifiedEmail : '',
+            $userGold: user.gold ? config.library.booleans.gold : '',
+            $profileId: user.profile.id,
+            $profileTitle: user.profile.title,
+            $profileDescription: user.profile.description,
+            $profileOver18: user.profile.over18 ? config.library.booleans.over18 : ''
+        });
+    }
+
+    if(post) {
+        Object.assign(vars, {
             $postId: post.id,
-        $postTitle: (post.title || '').slice(0, config.patterns.titleLength),
+            $postTitle: (post.title || '').slice(0, config.library.titleLength),
-        $postUser: post.user,
+            $postUser: post.user || user.user,
             $postDate: dateFns.format(post.datetime, dateFormat),
-        $postIndex: post.index + config.patterns.indexOffset,
+            $postIndex: post.index + config.library.indexOffset,
             $host: post.host.label
-    };
+        });
 
         if(post.content.album) {
             Object.assign(vars, {
                 $albumId: post.content.album.id,
-            $albumTitle: (post.content.album.title || '').slice(0, config.patterns.titleLength),
+                $albumTitle: (post.content.album.title || '').slice(0, config.library.titleLength),
                 $albumDescription: post.content.album.description,
                 $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
             });
         }
+    }
 
     if(item) {
         Object.assign(vars, {
             $itemId: item.id,
-            $itemTitle: (item.title || '').slice(0, config.patterns.titleLength),
+            $itemTitle: (item.title || '').slice(0, config.library.titleLength),
             $itemDescription: item.description,
             $itemDate: dateFns.format(item.datetime, dateFormat),
-            $itemIndex: item.index + config.patterns.indexOffset,
+            $itemIndex: item.index + config.library.indexOffset,
-            $extracted: item.extracted ? config.patterns.extractedLabel : '',
+            $extracted: item.extracted ? config.library.booleans.extracted : '',
-            $ext: extensions[item.type]
+            $ext: item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)
         });
     }
 
     return Object.entries(vars).reduce((acc, [key, value], index) =>  {
-        // strip slashes for filesystem compatability
+        // substitute slashes for filesystem compatability
-        value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
+        value = (value || '').toString().replace(/\//g, config.library.slashSubstitute);
 
         return acc.replace(key, value);
-    }, path);
+    }, pattern);
 };
 
 module.exports = interpolate;

save.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const fs = require('fs-extra');
+
+function save(filepath, stream) {
+    const file = fs.createWriteStream(filepath);
+
+    return new Promise((resolve, reject) => {
+        stream.pipe(file).on('error', error => {
+            reject(error);
+        }).on('finish', () => {
+            console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
+
+            resolve(filepath);
+        });
+    });
+};
+
+module.exports = save;

textToStream.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const Readable = require('stream').Readable;
+
+function textToStream(text) {
+    const stream = new Readable();
+
+    stream.push(text);
+    stream.push(null);
+
+    return stream;
+};
+
+module.exports = textToStream;