Flow and modularization refactor. Added duplicates option and applying limit after fetch.

app.js

@@ -6,55 +6,21 @@ const fs = require('fs-extra');
 const yargs = require('yargs').argv;
 const snoowrap = require('snoowrap');
 
-const curateSubmissions = require('./curate/submissions.js');
-const curateUser = require('./curate/user.js');
-const methods = require('./methods/methods.js');
-const interpolate = require('./interpolate.js');
-const fetchItem = require('./fetchItem.js');
-const fetchContent = require('./fetchContent.js');
-const save = require('./save.js');
-const textToStream = require('./textToStream.js');
-
 const reddit = new snoowrap(config.reddit.api);
 
-function saveProfileDetails(user) {
+const curateSubmissions = require('./curate/submissions.js');
-    if(config.library.profile.image) {
+const curateUser = require('./curate/user.js');
-        // pass profile image as item to interpolate extension variable
-        const filepath = interpolate(config.library.profile.image, user, null, {
-            url: user.profile.image
-        });
 
-        fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
+const methods = require('./methods/methods.js');
-            console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
+const interpolate = require('./interpolate.js');
-        });
-    }
 
-    if(config.library.profile.description) {
+const fetchInfo = require('./fetch/info.js');
-        if(user.profile.description) {
+const fetchContent = require('./fetch/content.js');
-            const filepath = interpolate(config.library.profile.description, user);
-            const stream = textToStream(user.profile.description);
 
-            save(filepath, stream).catch(error => {
+const save = require('./save/save.js');
-                console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
+const saveProfileDetails = require('./save/profileDetails.js');
-            });
-        } else {
-            console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
-        }
-    }
-};
 
-function getSubmissions(users, sort, limit) {
+const limit = yargs.limit || config.fetch.limit;
-    return users.reduce((chain, user) => {
-        return chain.then(acc => {
-            return reddit.getUser(user).getSubmissions({
-                sort: sort,
-                limit: limit
-            }).then(submissions => {
-                return acc.concat(submissions);
-            });
-        });
-    }, Promise.resolve([]));
-};
 
 if(!yargs.user && typeof yargs.users !== 'string') {
     return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user=[user], or multiple users with --users=[user1,user2] or --user=[user1] --user=[user2]');
@@ -62,50 +28,28 @@ if(!yargs.user && typeof yargs.users !== 'string') {
 
 const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
 
-users.forEach(user => {
+users.forEach(username => {
     return Promise.resolve().then(() => {
-        // get reddit profile
+        return reddit.getUser(username).fetch().then(curateUser);
-        return reddit.getUser(user).fetch().then(curateUser);
     }).then(user => {
         return saveProfileDetails(user);
-        // get submissions
+    }).then(user => {
-    }).catch(error => {
+        return reddit.getUser(username).getSubmissions({
-        return console.log('\x1b[33m%s\x1b[0m', error);
+            sort: yargs.sort || config.fetch.sort
-    });
+        }).then(submissions => ({
-});
+            user,
-
+            submissions
-/*
-Promise.resolve().then(() => {
-    if(yargs.user || yargs.users) {
-        const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
-
-        return Promise.resolve().then(() => {
-            if(config.library.profile) {
-                return getProfiles(users);
-            }
-        }).then(() => {
-            return getSubmissions(users, yargs.sort || config.reddit.sort, yargs.limit === undefined ? config.reddit.limit : yargs.limit);
-        });
-    }
-
-    return Promise.reject('Please supply at least one user with one or multiple --user, or --users!');
-}).then(submissions => {
-    return Promise.all(curate(submissions).reduce((acc, post) => {
-        if(post.host && methods[post.host.method]) {
-            acc = acc.concat(methods[post.host.method](post).then(content => {
-                post.content = content;
-
-                return post;
         }));
-        } else {
+    }).then(({user, submissions}) => {
-            console.log('\x1b[33m%s\x1b[0m', `"${post.title}": '${post.url}' not supported :(`);
+        const posts = curateSubmissions(submissions).slice(0, limit);
-        }
 
-        return acc;
+        return fetchInfo(posts).then(info => ({
-    }, []));
+            user,
-}).then(posts => {
+            posts
-    return fetchContent(posts);
+        }));
+    }).then(({user, posts}) => {
+        return fetchContent(posts, user);
     }).catch(error => {
-    return console.log('\x1b[31m%s\x1b[0m', error);
+        return console.log(error);
+    });
 });
-*/

config/default.js

@@ -10,8 +10,8 @@ module.exports = {
             video: '$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext'
         },
         profile: {
-            image: 'profile$ext',
+            image: '$userCreated - profile$ext',
-            description: 'profile ($userVerified$userVerifiedEmail$userGold)'
+            description: '$userCreated - profile ($userVerified$userVerifiedEmail$userGold$profileOver18)'
         },
         booleans: {
             extracted: 'extracted-',
@@ -25,9 +25,12 @@ module.exports = {
         indexOffset: 1,
         slashSubstitute: '#',
     },
-    reddit: {
+    fetch: {
         sort: 'top',
         limit: 1000,
+        ignoreDuplicates: true
+    },
+    reddit: {
         api: {
             userAgent: 'wat',
             clientId: 'VPquALMpTGl3ag',

curate/submissions.js

@@ -1,10 +1,19 @@
 'use strict';
 
+const config = require('config');
 const dissectLink = require('../dissectLink.js');
 
 function curateSubmissions(submissions) {
-    return submissions.map((submission, index) => {
+    const processed = new Set();
-        return {
+
+    return submissions.reduce((acc, submission, index) => {
+        if(config.fetch.ignoreDuplicates && processed.has(submission.url)) {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
+
+            return acc;
+        }
+
+        const curatedSubmission = {
             id: submission.id,
             index: index,
             title: submission.title,
@@ -16,7 +25,11 @@ function curateSubmissions(submissions) {
             subreddit: submission.subreddit.display_name,
             host: dissectLink(submission.url)
         };
-    });
+
+        processed.add(submission.url);
+
+        return acc.concat(curatedSubmission);
+    }, []);
 };
 
 module.exports = curateSubmissions;

curate/user.js

@@ -3,8 +3,6 @@
 const path = require('path');
 
 function curateUser(user) {
-    console.log(user);
-
     return {
         id: user.id,
         name: user.name,

fetch/content.js

@@ -4,12 +4,12 @@ const fs = require('fs-extra');
 const path = require('path');
 const config = require('config');
 
-const fetchItem = require('./fetchItem');
+const fetchItem = require('./item.js');
-const save = require('./save.js');
+const interpolate = require('../interpolate.js');
-const interpolate = require('./interpolate.js');
+const save = require('../save/save.js');
-const textToStream = require('./textToStream.js');
+const textToStream = require('../save/textToStream.js');
 
-module.exports = function(posts) {
+module.exports = function(posts, user) {
     return Promise.all(posts.map(post => {
         return Promise.resolve().then(() => {
             return Promise.all(post.content.items.map((item, index) => {
@@ -26,7 +26,7 @@ module.exports = function(posts) {
         }).then(items => {
             return Promise.all(items.map(item => {
                 const type = item.type.split('/')[0];
-                const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
+                const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
 
                 return Promise.resolve().then(() => {
                     return fs.ensureDir(path.dirname(filepath));

fetch/info.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const methods = require('../methods/methods.js');
+
+function fetchInfo(posts) {
+    return Promise.all(posts.reduce((acc, post) => {
+        if(post.host && methods[post.host.method]) {
+            acc = acc.concat(methods[post.host.method](post).then(content => {
+                post.content = content;
+
+                return post;
+            }));
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.title}' - ${post.url}`);
+        }
+
+        return acc;
+    }, []));
+};
+
+module.exports = fetchInfo;

interpolate.js

@@ -64,7 +64,7 @@ function interpolate(pattern, user, post, item) {
             $itemDate: dateFns.format(item.datetime, dateFormat),
             $itemIndex: item.index + config.library.indexOffset,
             $extracted: item.extracted ? config.library.booleans.extracted : '',
-            $ext: item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)
+            $ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname))
         });
     }
 

methods/imgurAlbum.js

@@ -10,7 +10,7 @@ function imgurAlbum(post) {
             'Authorization': `Client-ID ${config.methods.imgur.clientId}`
         }
     }).then(res => res.json()).then(res => {
-        const extract = config.patterns.album.extractSingleItem && res.data.images.length === 1;
+        const extract = config.library.album.extractSingleItem && res.data.images.length === 1;
 
         return {
             album: extract ? null : {

save.js

@@ -1,19 +0,0 @@
-'use strict';
-
-const fs = require('fs-extra');
-
-function save(filepath, stream) {
-    const file = fs.createWriteStream(filepath);
-
-    return new Promise((resolve, reject) => {
-        stream.pipe(file).on('error', error => {
-            reject(error);
-        }).on('finish', () => {
-            console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
-
-            resolve(filepath);
-        });
-    });
-};
-
-module.exports = save;

save/profileDetails.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const config = require('config');
+
+const interpolate = require('../interpolate.js');
+const fetchItem = require('../fetch/item.js');
+const textToStream = require('./textToStream.js');
+const save = require('./save.js');
+
+function saveProfileDetails(user) {
+    if(config.library.profile.image) {
+        const filepath = interpolate(config.library.profile.image, user, null, {
+            // pass profile image as item to interpolate extension variable
+            url: user.profile.image
+        });
+
+        fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
+            console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
+        });
+    }
+
+    if(config.library.profile.description) {
+        if(user.profile.description) {
+            const filepath = interpolate(config.library.profile.description, user);
+            const stream = textToStream(user.profile.description);
+
+            save(filepath, stream).catch(error => {
+                console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
+            });
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
+        }
+    }
+
+    return user;
+};
+
+module.exports = saveProfileDetails;

save/save.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+
+function save(filepath, stream) {
+    return Promise.resolve().then(() => {
+        return fs.ensureDir(path.dirname(filepath));
+    }).then(() => {
+        const file = fs.createWriteStream(filepath);
+
+        return new Promise((resolve, reject) => {
+            stream.pipe(file).on('error', error => {
+                reject(error);
+            }).on('finish', () => {
+                console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
+
+                resolve(filepath);
+            });
+        });
+    });
+};
+
+module.exports = save;