Refactored info content fetching and saving to handle users object and utilize async/await.

.eslintrc

@@ -0,0 +1,8 @@
+{
+    "extends": "airbnb-base",
+    "rules": {
+        "no-console": 0,
+        "indent": ["error", 4],
+        "max-len": [2, {"code": 125, "tabWidth": 4, "ignoreUrls": true}]
+    }
+}

package.json

@@ -44,5 +44,10 @@
     "snoowrap": "^1.15.2",
     "url-pattern": "^1.0.3",
     "yargs": "^11.0.0"
+  },
+  "devDependencies": {
+    "eslint": "^4.19.1",
+    "eslint-config-airbnb-base": "^12.1.0",
+    "eslint-plugin-import": "^2.12.0"
   }
 }

src/app.js

@@ -5,6 +5,8 @@ const util = require('util');
 const fs = require('fs-extra');
 const snoowrap = require('snoowrap');
 const omit = require('object.omit');
+const exiftool = require('node-exiftool');
+const exiftoolBin = require('dist-exiftool');
 
 require('promise.prototype.finally').shim();
 require('array.prototype.flatten').shim();
@@ -26,6 +28,8 @@ if(!(args.users && args.users.length) && !(args.posts && args.posts.length)) {
     return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user <user> or one post with --post <post-id>. See --help for more options.');
 }
 
+const ep = new exiftool.ExiftoolProcess(exiftoolBin);
+
 Promise.resolve().then(async () => {
     const initUsers = args.users ? args.users.reduce((acc, username) => ({...acc, [username]: {name: username, posts: []}}), {}) : {};
     let userPosts = await getUserPosts(initUsers);
@@ -37,15 +41,9 @@ Promise.resolve().then(async () => {
     const curatedUserPosts = curatePosts(userPosts, args);
     const infoUserPosts = await attachContentInfo(curatedUserPosts);
 
-    console.log(util.inspect(infoUserPosts, {depth: 10}));
+    await ep.open();
-
+    await Promise.all(Object.values(infoUserPosts).map(user => fetchContent(user, ep)));
-    /*
+    await ep.close();
-    for(const user of args.users) {
-        console.log(user);
-    }
-    */
-
-    // return fetchContent(posts, userPosts);
 }).catch(error => {
     return console.error(error);
 });

src/fetch/content.js

@@ -1,8 +1,6 @@
-'use strict';
-
 const fs = require('fs-extra');
-const path = require('path');
 const config = require('config');
+const Promise = require('bluebird');
 
 const fetchItem = require('./item.js');
 const interpolate = require('../interpolate.js');
@@ -11,72 +9,62 @@ const textToStream = require('../save/textToStream.js');
 const saveMeta = require('../save/meta.js');
 const mux = require('../save/mux.js');
 
-const exiftool = require('node-exiftool');
+async function getStreams(item, post) {
-const exiftoolBin = require('dist-exiftool');
+    if(item.self) {
+        return [textToStream(item.text)];
+    }
 
-const ep = new exiftool.ExiftoolProcess(exiftoolBin);
+    const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
+    const streams = await Promise.map(sources, source => fetchItem(source, 0, post));
 
-module.exports = function(posts, users) {
+    if (streams.filter(stream => stream).length > 0) {
-    return Promise.resolve().then(() => {
+        return streams;
-        return ep.open();
+    }
-    }).then(() => {
-        // console.log(users);
-    }).then(() => {
-        return Promise.all(posts.map(post => {
-            return Promise.all(post.content.items.map((item, index) => {
-                item.index = index;
 
-                if(item.self) {
+    return null;
-                    return Object.assign({}, item, {stream: textToStream(item.text)});
+}
-                }
 
-                // some videos are delivered with separate audio and are fetched separately to be muxed later
+async function fetchContent(user, ep) {
-                const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
+    await Promise.map(user.posts, async (post) => {
+        const items = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
+            const item = { ...originalItem, index };
+            const streams = await getStreams(item, post);
 
-                return Promise.all(sources.map(source => {
+            // no streams, ignore item
-                    return fetchItem(source, 0, post);
+            if (streams.length <= 0) {
-                })).then(streams => {
+                return accItems;
-                    if(streams.filter(stream => stream).length > 0) {
+            }
-                        return Object.assign({}, item, {streams})
-                    }
 
-                    return null;
+            const type = item.type.split('/')[0];
-                });
+            const filepath = post.content.album
-            })).then(items => items.filter(item => item)).then(items => {
+                ? interpolate(config.library.album[type], user, post, item)
-                return Promise.all(items.map(item => {
+                : interpolate(config.library[type], user, post, item);
-                    const type = item.type.split('/')[0];
-                    const filepath = post.content.album ? interpolate(config.library.album[type], post.user, post, item) : interpolate(config.library[type], post.user, post, item);
 
-                    return Promise.resolve().then(() => {
+            const sourcePaths = await save(filepath, streams, item, post);
-                        return save(filepath, item.streams || item.stream, item, post);
-                    }).then(sourcePaths => {
-                        if(item.mux) {
-                            return mux(filepath, sourcePaths, item);
-                        }
-                    }).then(() => {
-                        const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
-                            const interpolatedValue = interpolate(value, post.user, post, item);
 
-                            if(interpolatedValue) {
+            if (item.mux) {
-                                acc[key] = interpolatedValue;
+                await mux(filepath, sourcePaths, item);
-                            }
+            }
 
-                            return acc;
+            const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
-                        }, {});
+                const interpolatedValue = interpolate(value, user, post, item);
 
-                        if(Object.keys(meta).length > 0) {
+                return interpolatedValue ? { ...acc, [key]: interpolatedValue } : acc;
-                            return saveMeta(filepath, meta, ep);
+            }, {});
-                        }
-                    });
-                })).then(result => {
-                    const filename = interpolate(config.library.index.file, post.user, post);
-                    const entry = interpolate(config.library.index.entry, post.user, post, null, false) + '\n';
 
-                    return fs.appendFile(filename, config.library.index.unique ? `${post.hash} ${entry}` : entry);
+            if (Object.keys(meta).length > 0) {
-                });
+                await saveMeta(filepath, meta, ep);
-            });
+            }
-        }));
+
-    }).finally(() => {
+            return sourcePaths;
-        return ep.close();
+        }, []);
+
+        console.log(items);
+
+        const filename = interpolate(config.library.index.file, user, post);
+        const entry = `${interpolate(config.library.index.entry, user, post, null, false)}\n`;
+
+        await fs.appendFile(filename, config.library.index.unique ? `${post.hash} ${entry}` : entry);
     });
-};
+}
+module.exports = fetchContent;

src/fetch/info.js

@@ -14,7 +14,7 @@ const attachContentInfo = users => {
         }
 
         try {
-            return [...accPosts, await methods[post.host.method](post)];
+            return [...accPosts, {...post, content: await methods[post.host.method](post)}];
         } catch(error) {
             console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);