Flow and modularization refactor. Added duplicates option and applying limit after fetch.

fetch/content.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const config = require('config');
+
+const fetchItem = require('./item.js');
+const interpolate = require('../interpolate.js');
+const save = require('../save/save.js');
+const textToStream = require('../save/textToStream.js');
+
+module.exports = function(posts, user) {
+    return Promise.all(posts.map(post => {
+        return Promise.resolve().then(() => {
+            return Promise.all(post.content.items.map((item, index) => {
+                item.index = index;
+
+                if(item.self) {
+                    return Object.assign({}, item, {stream: textToStream(item.text)});
+                }
+
+                return fetchItem(item.url, 0).then(stream => {
+                    return Object.assign({}, item, {stream});
+                });
+            }));
+        }).then(items => {
+            return Promise.all(items.map(item => {
+                const type = item.type.split('/')[0];
+                const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
+
+                return Promise.resolve().then(() => {
+                    return fs.ensureDir(path.dirname(filepath));
+                }).then(() => {
+                    return save(filepath, item.stream)
+                });
+            }));
+        });
+    }));
+};

fetch/info.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const methods = require('../methods/methods.js');
+
+function fetchInfo(posts) {
+    return Promise.all(posts.reduce((acc, post) => {
+        if(post.host && methods[post.host.method]) {
+            acc = acc.concat(methods[post.host.method](post).then(content => {
+                post.content = content;
+
+                return post;
+            }));
+        } else {
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.title}' - ${post.url}`);
+        }
+
+        return acc;
+    }, []));
+};
+
+module.exports = fetchInfo;

fetch/item.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const fetch = require('node-fetch');
+
+function fetchItem(url, attempt) {
+    function retry(error) {
+        console.log(error);
+
+        if(attempt < 3) {
+            console.log('Retrying...');
+
+            return fetchItem(url, ++attempt);
+        }
+    };
+
+    return fetch(url).then(res => {
+        return res.ok ? res : Promise.reject(`Failed to fetch ${url}`);
+    }).then(res => {
+        console.log(`Fetched '${url}'`);
+
+        return res.body;
+    }).catch(retry);
+};
+
+module.exports = fetchItem;