Moved source into dedicated directory.

src/app.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const config = require('config');
+const util = require('util');
+const fs = require('fs-extra');
+const yargs = require('yargs').argv;
+const snoowrap = require('snoowrap');
+
+const reddit = new snoowrap(config.reddit.api);
+
+const curateSubmissions = require('./curate/submissions.js');
+const curateUser = require('./curate/user.js');
+
+const methods = require('./methods/methods.js');
+const interpolate = require('./interpolate.js');
+
+const fetchInfo = require('./fetch/info.js');
+const fetchContent = require('./fetch/content.js');
+
+const save = require('./save/save.js');
+const saveProfileDetails = require('./save/profileDetails.js');
+
+const limit = yargs.limit || config.fetch.limit;
+
+if(!yargs.user && typeof yargs.users !== 'string') {
+    return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user=[user], or multiple users with --users=[user1,user2] or --user=[user1] --user=[user2]');
+}
+
+const users = yargs.users ? yargs.users.split(',') : [].concat(yargs.user);
+
+users.forEach(username => {
+    return Promise.resolve().then(() => {
+        return reddit.getUser(username).fetch().then(curateUser);
+    }).then(user => {
+        return saveProfileDetails(user);
+    }).then(user => {
+        return reddit.getUser(username).getSubmissions({
+            sort: yargs.sort || config.fetch.sort,
+            limit: Infinity
+        }).then(submissions => ({
+            user,
+            submissions
+        }));
+    }).then(({user, submissions}) => {
+        const posts = curateSubmissions(submissions).slice(0, limit);
+
+        return fetchInfo(posts).then(info => ({
+            user,
+            posts
+        }));
+    }).then(({user, posts}) => {
+        return fetchContent(posts, user);
+    }).catch(error => {
+        return console.log(error);
+    });
+});

src/curate/submissions.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const config = require('config');
+const dissectLink = require('../dissectLink.js');
+
+function curateSubmissions(submissions) {
+    const processed = new Set();
+
+    return submissions.reduce((acc, submission, index) => {
+        if(config.fetch.ignoreDuplicates && processed.has(submission.url)) {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
+
+            return acc;
+        }
+
+        const curatedSubmission = {
+            id: submission.id,
+            index: index,
+            title: submission.title,
+            text: submission.selftext,
+            user: submission.author.name,
+            permalink: submission.permalink,
+            url: submission.url,
+            datetime: new Date(submission.created_utc * 1000),
+            subreddit: submission.subreddit.display_name,
+            host: dissectLink(submission.url)
+        };
+
+        processed.add(submission.url);
+
+        return acc.concat(curatedSubmission);
+    }, []);
+};
+
+module.exports = curateSubmissions;

src/curate/user.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const path = require('path');
+
+function curateUser(user) {
+    return {
+        id: user.id,
+        name: user.name,
+        created: new Date(user.created_utc * 1000),
+        gold: user.is_gold,
+        verified: user.verified,
+        verifiedEmail: user.has_verified_email,
+        profile: {
+            id: user.subreddit.display_name.name,
+            title: user.subreddit.display_name.title,
+            image: user.subreddit.display_name.icon_img,
+            banner: user.subreddit.display_name.banner_img,
+            description: user.subreddit.display_name.public_description,
+            over18: user.subreddit.display_name.over_18
+        }
+    };
+};
+
+module.exports = curateUser;

src/dissectLink.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const urlPattern = require('url-pattern');
+
+const hosts = [{
+    method: 'self',
+    label: 'self',
+    pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/')
+}, {
+    method: 'reddit',
+    label: 'reddit',
+    pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext')
+}, {
+    method: 'imgurImage',
+    label: 'imgur',
+    pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
+}, {
+    method: 'imgurAlbum',
+    label: 'imgur',
+    pattern: new urlPattern('http(s)\\://(m.)imgur.com/:type/:id')
+}, {
+    method: 'gfycat',
+    label: 'gfycat',
+    pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)')
+}, {
+    method: 'eroshare',
+    label: 'eroshare',
+    pattern: new urlPattern('http(s)\\://eroshare.com/:id')
+}];
+
+module.exports = function dissectLink(url) {
+    return hosts.reduce((acc, host) => {
+        if(acc) {
+            return acc;
+        }
+
+        const match = host.pattern.match(url);
+
+        if(match) {
+            return Object.assign(match, {
+                url: url,
+                method: host.method,
+                label: host.label
+            });
+        }
+
+        return null;
+    }, null);
+};

src/fetch/content.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const config = require('config');
+
+const fetchItem = require('./item.js');
+const interpolate = require('../interpolate.js');
+const save = require('../save/save.js');
+const textToStream = require('../save/textToStream.js');
+
+module.exports = function(posts, user) {
+    return Promise.all(posts.map(post => {
+        return Promise.resolve().then(() => {
+            return Promise.all(post.content.items.map((item, index) => {
+                item.index = index;
+
+                if(item.self) {
+                    return Object.assign({}, item, {stream: textToStream(item.text)});
+                }
+
+                return fetchItem(item.url, 0).then(stream => {
+                    return Object.assign({}, item, {stream});
+                });
+            }));
+        }).then(items => {
+            return Promise.all(items.map(item => {
+                const type = item.type.split('/')[0];
+                const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
+
+                return Promise.resolve().then(() => {
+                    return fs.ensureDir(path.dirname(filepath));
+                }).then(() => {
+                    return save(filepath, item.stream)
+                });
+            }));
+        });
+    }));
+};

src/fetch/info.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const methods = require('../methods/methods.js');
+
+function fetchInfo(posts) {
+    return Promise.all(posts.reduce((acc, post) => {
+        if(post.host && methods[post.host.method]) {
+            acc = acc.concat(methods[post.host.method](post).then(content => {
+                post.content = content;
+
+                return post;
+            }));
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.title}' - ${post.url}`);
+        }
+
+        return acc;
+    }, []));
+};
+
+module.exports = fetchInfo;

src/fetch/item.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const fetch = require('node-fetch');
+
+function fetchItem(url, attempt) {
+    function retry(error) {
+        console.log(error);
+
+        if(attempt < 3) {
+            console.log('Retrying...');
+
+            return fetchItem(url, ++attempt);
+        }
+    };
+
+    return fetch(url).then(res => {
+        return res.ok ? res : Promise.reject(`Failed to fetch ${url}`);
+    }).then(res => {
+        console.log(`Fetched '${url}'`);
+
+        return res.body;
+    }).catch(retry);
+};
+
+module.exports = fetchItem;

src/interpolate.js

@@ -0,0 +1,79 @@
+'use strict';
+
+const config = require('config');
+const path = require('path');
+const url = require('url');
+const dateFns = require('date-fns');
+
+const extensions = {
+    'image/jpeg': '.jpg',
+    'image/gif': '.gif',
+    'video/mp4': '.mp4',
+    'video/webm': '.webm'
+};
+
+function interpolate(pattern, user, post, item) {
+    const dateFormat = config.library.dateFormat || 'YYYYMMDD';
+    const vars = {};
+
+    if(config.library.base) {
+        pattern = path.join(config.library.base, pattern);
+    }
+
+    if(user) {
+        Object.assign(vars, {
+            $user: user.name,
+            $username: user.name,
+            $userId: user.id,
+            $userCreated: dateFns.format(user.created, dateFormat),
+            $userVerified: user.verified ? config.library.booleans.verified : '',
+            $userVerifiedEmail: user.verifiedEmail ? config.library.booleans.verifiedEmail : '',
+            $userGold: user.gold ? config.library.booleans.gold : '',
+            $profileId: user.profile.id,
+            $profileTitle: user.profile.title,
+            $profileDescription: user.profile.description,
+            $profileOver18: user.profile.over18 ? config.library.booleans.over18 : ''
+        });
+    }
+
+    if(post) {
+        Object.assign(vars, {
+            $postId: post.id,
+            $postTitle: (post.title || '').slice(0, config.library.titleLength),
+            $postUser: post.user || user.user,
+            $postDate: dateFns.format(post.datetime, dateFormat),
+            $postIndex: post.index + config.library.indexOffset,
+            $host: post.host.label
+        });
+
+        if(post.content.album) {
+            Object.assign(vars, {
+                $albumId: post.content.album.id,
+                $albumTitle: (post.content.album.title || '').slice(0, config.library.titleLength),
+                $albumDescription: post.content.album.description,
+                $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
+            });
+        }
+    }
+
+    if(item) {
+        Object.assign(vars, {
+            $itemId: item.id,
+            $itemTitle: (item.title || '').slice(0, config.library.titleLength),
+            $itemDescription: item.description,
+            $itemDate: dateFns.format(item.datetime, dateFormat),
+            $itemIndex: item.index + config.library.indexOffset,
+            $extracted: item.extracted ? config.library.booleans.extracted : '',
+            $ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname))
+        });
+    }
+
+    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
+        // substitute slashes for filesystem compatability
+        value = (value || '').toString().replace(/\//g, config.library.slashSubstitute);
+
+        return acc.replace(key, value);
+    }, pattern);
+};
+
+module.exports = interpolate;

src/methods/eroshare.js

@@ -0,0 +1,44 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function eroshare(post) {
+    return fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${post.host.id}`).then(res => {
+        if(res.ok) {
+            return res.text()
+        }
+
+        return Promise.reject(`Unable to recover Eroshare video '${post.host.id}' :(`);
+    }).then(res => {
+        const data = JSON.parse(res.match(/var album = .*/)[0].slice(12, -1));
+        const extract = config.patterns.album.extractSingleItem && data.items.length === 1;
+
+        return {
+            album: extract ? null : {
+                id: data.slug,
+                title: data.title,
+                datetime: new Date(data.created_at)
+            },
+            items: data.items.map(item => {
+                return {
+                    extracted: extract,
+                    id: item.slug,
+                    url: item.type === 'Image' ? item.url_full_protocol : item.url_mp4,
+                    title: data.title,
+                    description: item.description,
+                    type: item.type === 'Image' ? 'image/jpeg' : 'video/mp4',
+                    datetime: new Date(data.created_at),
+                    width: data.width,
+                    height: data.height,
+                    original: item
+                };
+            })
+        };
+    }).catch(error => {
+        console.log('\x1b[33m%s\x1b[0m', error);
+    });
+};
+
+module.exports = eroshare;

src/methods/gfycat.js

@@ -0,0 +1,26 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function gfycat(post) {
+    return fetch(`https://gfycat.com/cajax/get/${post.host.id}`).then(res => res.json()).then(res => {
+        return {
+            album: null,
+            items: [{
+                id: res.gfyItem.gfyName,
+                url: res.gfyItem.webmUrl,
+                title: res.gfyItem.title,
+                description: res.gfyItem.description,
+                type: 'video/webm',
+                datetime: new Date(res.gfyItem.createDate * 1000),
+                original: res.gfyItem
+            }]
+        };
+    }).catch(error => {
+        console.error(error);
+    });
+};
+
+module.exports = gfycat;

src/methods/imgurAlbum.js

@@ -0,0 +1,44 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function imgurAlbum(post) {
+    return fetch(`https://api.imgur.com/3/album/${post.host.id}`, {
+        headers: {
+            'Authorization': `Client-ID ${config.methods.imgur.clientId}`
+        }
+    }).then(res => res.json()).then(res => {
+        const extract = config.library.album.extractSingleItem && res.data.images.length === 1;
+
+        if(extract) {
+            console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.data.link}`);
+        }
+
+        return {
+            album: extract ? null : {
+                id: res.data.id,
+                url: res.data.link,
+                title: res.data.title,
+                description: res.data.description,
+                datetime: new Date(res.data.datetime * 1000),
+                original: res.data
+            },
+            items: res.data.images.map(item => ({
+                extracted: extract,
+                id: item.id,
+                url: item.animated ? item.mp4 : item.link,
+                title: item.title || (extract ? res.data.title : null),
+                description: item.description || (extract ? res.data.description : null),
+                type: item.animated ? 'video/mp4' : item.type,
+                datetime: item.datetime * 1000,
+                original: item
+            }))
+        };
+    }).catch(error => {
+        console.error(error);
+    });
+};
+
+module.exports = imgurAlbum;

src/methods/imgurImage.js

@@ -0,0 +1,30 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function imgurImage(post) {
+    return fetch(`https://api.imgur.com/3/image/${post.host.id}`, {
+        headers: {
+            'Authorization': `Client-ID ${config.methods.imgur.clientId}`
+        }
+    }).then(res => res.json()).then(res => {
+        return {
+            album: null,
+            items: [{
+                id: res.data.id,
+                url: res.data.animated ? res.data.mp4 : res.data.link,
+                title: res.data.title,
+                description: res.data.description,
+                type: res.data.animated ? 'video/mp4' : res.data.type,
+                datetime: new Date(res.data.datetime * 1000),
+                original: res.data
+            }]
+        };
+    }).catch(error => {
+        console.error(error);
+    });
+};
+
+module.exports = imgurImage;

src/methods/methods.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const self = require('./self.js');
+const reddit = require('./reddit.js');
+const imgurImage = require('./imgurImage.js');
+const imgurAlbum = require('./imgurAlbum.js');
+const gfycat = require('./gfycat.js');
+const eroshare = require('./eroshare.js');
+
+module.exports = {
+    self: self,
+    reddit: reddit,
+    imgurImage: imgurImage,
+    imgurAlbum: imgurAlbum,
+    gfycat: gfycat,
+    eroshare: eroshare
+};

src/methods/reddit.js

@@ -0,0 +1,21 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function reddit(post) {
+    return Promise.resolve({
+        album: null,
+        items: [{
+            id: post.host.id || post.id,
+            url: post.url,
+            title: post.title,
+            datetime: post.datetime,
+            type: 'image/jpeg',
+            original: post
+        }]
+    });
+};
+
+module.exports = reddit;

src/methods/self.js

@@ -0,0 +1,23 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function self(post) {
+    return Promise.resolve({
+        album: null,
+        items: [{
+            id: post.id,
+            url: post.url,
+            title: post.title,
+            text: post.text,
+            datetime: post.datetime,
+            type: 'text/plain',
+            self: true,
+            original: post
+        }]
+    });
+};
+
+module.exports = self;

src/save/profileDetails.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const config = require('config');
+
+const interpolate = require('../interpolate.js');
+const fetchItem = require('../fetch/item.js');
+const textToStream = require('./textToStream.js');
+const save = require('./save.js');
+
+function saveProfileDetails(user) {
+    if(config.library.profile.image) {
+        const filepath = interpolate(config.library.profile.image, user, null, {
+            // pass profile image as item to interpolate extension variable
+            url: user.profile.image
+        });
+
+        fetchItem(user.profile.image).then(stream => save(filepath, stream)).catch(error => {
+            console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
+        });
+    }
+
+    if(config.library.profile.description) {
+        if(user.profile.description) {
+            const filepath = interpolate(config.library.profile.description, user);
+            const stream = textToStream(user.profile.description);
+
+            save(filepath, stream).catch(error => {
+                console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
+            });
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
+        }
+    }
+
+    return user;
+};
+
+module.exports = saveProfileDetails;

src/save/save.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+
+function save(filepath, stream) {
+    return Promise.resolve().then(() => {
+        return fs.ensureDir(path.dirname(filepath));
+    }).then(() => {
+        const file = fs.createWriteStream(filepath);
+
+        return new Promise((resolve, reject) => {
+            stream.pipe(file).on('error', error => {
+                reject(error);
+            }).on('finish', () => {
+                console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
+
+                resolve(filepath);
+            });
+        });
+    });
+};
+
+module.exports = save;

src/save/textToStream.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const Readable = require('stream').Readable;
+
+function textToStream(text) {
+    const stream = new Readable();
+
+    stream.push(text);
+    stream.push(null);
+
+    return stream;
+};
+
+module.exports = textToStream;