Added support for muxing streams, specifically for reddit-hosted videos (now also supported).

package-lock.json

@@ -35,6 +35,14 @@
       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
     },
+    "async": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz",
+      "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==",
+      "requires": {
+        "lodash": "4.17.5"
+      }
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -209,6 +217,15 @@
       "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
       "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
     },
+    "define-properties": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
+      "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
+      "requires": {
+        "foreach": "2.0.5",
+        "object-keys": "1.0.11"
+      }
+    },
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -276,6 +293,28 @@
       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
       "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
     },
+    "es-abstract": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz",
+      "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==",
+      "requires": {
+        "es-to-primitive": "1.1.1",
+        "function-bind": "1.1.1",
+        "has": "1.0.1",
+        "is-callable": "1.1.3",
+        "is-regex": "1.0.4"
+      }
+    },
+    "es-to-primitive": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
+      "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
+      "requires": {
+        "is-callable": "1.1.3",
+        "is-date-object": "1.0.1",
+        "is-symbol": "1.0.1"
+      }
+    },
     "execa": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@@ -330,6 +369,20 @@
         "locate-path": "2.0.0"
       }
     },
+    "fluent-ffmpeg": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+      "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
+      "requires": {
+        "async": "2.6.0",
+        "which": "1.3.0"
+      }
+    },
+    "foreach": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
+      "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
+    },
     "forever-agent": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -355,6 +408,11 @@
         "universalify": "0.1.1"
       }
     },
+    "function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+    },
     "get-caller-file": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz",
@@ -397,6 +455,14 @@
       "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.0.tgz",
       "integrity": "sha512-0kZ1XcoelFOLEjEtvWAZyq/1S55eDSieWEJwme311MNVNcRpvjlr2zA66kBV6WAB8C1XI1p1cXCnFPqd1BxlPg=="
     },
+    "has": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz",
+      "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=",
+      "requires": {
+        "function-bind": "1.1.1"
+      }
+    },
     "hawk": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
@@ -446,16 +512,39 @@
       "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
       "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY="
     },
+    "is-callable": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz",
+      "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
+    },
+    "is-date-object": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
+      "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
+    },
     "is-fullwidth-code-point": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
       "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
     },
+    "is-regex": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
+      "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
+      "requires": {
+        "has": "1.0.1"
+      }
+    },
     "is-stream": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
       "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
     },
+    "is-symbol": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
+      "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI="
+    },
     "is-typedarray": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@@ -624,6 +713,11 @@
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
       "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
     },
+    "object-keys": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz",
+      "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0="
+    },
     "options": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz",
@@ -711,6 +805,16 @@
         "harmony-reflect": "1.6.0"
       }
     },
+    "promise.prototype.finally": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.0.tgz",
+      "integrity": "sha512-7p/K2f6dI+dM8yjRQEGrTQs5hTQixUAdOGpMEA3+pVxpX5oHKRSKAXyLw9Q9HUWDTdwtoo39dSHGQtN90HcEwQ==",
+      "requires": {
+        "define-properties": "1.1.2",
+        "es-abstract": "1.11.0",
+        "function-bind": "1.1.1"
+      }
+    },
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",

package.json

@@ -26,9 +26,11 @@
     "config": "^1.30.0",
     "date-fns": "^1.29.0",
     "dist-exiftool": "^10.53.0",
+    "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^5.0.0",
     "node-exiftool": "^2.3.0",
     "node-fetch": "^2.1.2",
+    "promise.prototype.finally": "^3.1.0",
     "snoowrap": "^1.15.2",
     "url-pattern": "^1.0.3",
     "yargs": "^11.0.0"

src/app.js

@@ -5,6 +5,7 @@ const util = require('util');
 const fs = require('fs-extra');
 const yargs = require('yargs').argv;
 const snoowrap = require('snoowrap');
+const promiseFinally = require('promise.prototype.finally');
 
 const reddit = new snoowrap(config.reddit.api);
 
@@ -20,6 +21,8 @@ const fetchContent = require('./fetch/content.js');
 const save = require('./save/save.js');
 const saveProfileDetails = require('./save/profileDetails.js');
 
+promiseFinally.shim();
+
 const limit = yargs.limit || config.fetch.limit;
 
 if(!yargs.user && typeof yargs.users !== 'string') {

src/curate/submissions.js

@@ -25,7 +25,7 @@ function curateSubmissions(submissions) {
             title: submission.title,
             text: submission.selftext,
             user: submission.author.name,
-            permalink: submission.permalink,
+            permalink: 'https://reddit.com' + submission.permalink,
             url: submission.url,
             datetime: new Date(submission.created_utc * 1000),
             subreddit: submission.subreddit.display_name,

src/dissectLink.js

@@ -7,13 +7,17 @@ const hosts = [{
     label: 'self',
     pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/')
 }, {
-    method: 'reddit',
+    method: 'redditImage',
     label: 'reddit',
     pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext')
 }, {
-    method: 'reddit',
+    method: 'redditImage',
     label: 'reddit',
-    pattern: new urlPattern('https\\://i.reddituploads.com/:id(?*)')
+    pattern: new urlPattern('http(s)\\://i.reddituploads.com/:id(?*)')
+}, {
+    method: 'redditVideo',
+    label: 'reddit',
+    pattern: new urlPattern('http(s)\\://v.redd.it/:id')
 }, {
     method: 'imgurImage',
     label: 'imgur',

src/fetch/content.js

@@ -9,6 +9,7 @@ const interpolate = require('../interpolate.js');
 const save = require('../save/save.js');
 const textToStream = require('../save/textToStream.js');
 const saveMeta = require('../save/meta.js');
+const mux = require('../save/mux.js');
 
 const exiftool = require('node-exiftool');
 const exiftoolBin = require('dist-exiftool');
@@ -20,19 +21,20 @@ module.exports = function(posts, user) {
         return ep.open();
     }).then(() => {
         return Promise.all(posts.map(post => {
-            return Promise.resolve().then(() => {
+            return Promise.all(post.content.items.map((item, index) => {
-                return Promise.all(post.content.items.map((item, index) => {
+                item.index = index;
-                    item.index = index;
 
-                    if(item.self) {
+                if(item.self) {
-                        return Object.assign({}, item, {stream: textToStream(item.text)});
+                    return Object.assign({}, item, {stream: textToStream(item.text)});
-                    }
+                }
 
-                    return fetchItem(item.url, 0).then(stream => {
+                // some videos are delivered with separate audio and are fetched separately to be muxed later
-                        return Object.assign({}, item, {stream});
+                const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
-                    });
+
-                }));
+                return Promise.all(sources.map(source => {
-            }).then(items => {
+                    return fetchItem(source, 0);
+                })).then(streams => Object.assign({}, item, {streams}));
+            })).then(items => {
                 return Promise.all(items.map(item => {
                     const type = item.type.split('/')[0];
                     const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
@@ -40,7 +42,11 @@ module.exports = function(posts, user) {
                     return Promise.resolve().then(() => {
                         return fs.ensureDir(path.dirname(filepath));
                     }).then(() => {
-                        return save(filepath, item.stream);
+                        return save(filepath, item);
+                    }).then(sourcePaths => {
+                        if(item.mux) {
+                            return mux(sourcePaths, filepath, item);
+                        }
                     }).then(() => {
                         const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
                             const interpolatedValue = interpolate(value, user, post, item);
@@ -59,9 +65,7 @@ module.exports = function(posts, user) {
                 }));
             });
         }));
-    }).then(() => {
+    }).finally(() => {
-        return ep.close();
-    }).catch(error => {
         return ep.close();
     });
 };

src/methods/methods.js

@@ -1,7 +1,8 @@
 'use strict';
 
 const self = require('./self.js');
-const reddit = require('./reddit.js');
+const redditImage = require('./redditImage.js');
+const redditVideo = require('./redditVideo.js');
 const imgurImage = require('./imgurImage.js');
 const imgurAlbum = require('./imgurAlbum.js');
 const gfycat = require('./gfycat.js');
@@ -9,7 +10,8 @@ const eroshare = require('./eroshare.js');
 
 module.exports = {
     self: self,
-    reddit: reddit,
+    redditImage: redditImage,
+    redditVideo: redditVideo,
     imgurImage: imgurImage,
     imgurAlbum: imgurAlbum,
     gfycat: gfycat,

src/methods/redditImage.js

@@ -4,7 +4,7 @@ const util = require('util');
 const config = require('config');
 const fetch = require('node-fetch');
 
-function reddit(post) {
+function redditImage(post) {
     return Promise.resolve({
         album: null,
         items: [{
@@ -18,4 +18,4 @@ function reddit(post) {
     });
 };
 
-module.exports = reddit;
+module.exports = redditImage;

src/methods/redditVideo.js

@@ -0,0 +1,38 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+const fs = require('fs-extra');
+
+function redditVideo(post) {
+    return fetch(`${post.permalink}.json`).then(res => res.json()).then(res => {
+        return res[0].data.children[0].data.media.reddit_video.fallback_url;
+    }).then(videoUrl => {
+        const audioUrl = videoUrl.split('/').slice(0, -1).join('/') + '/audio';
+
+        return fetch(audioUrl, {
+            method: 'HEAD'
+        }).then(res => {
+            const item = {
+                album: null,
+                items: [{
+                    id: post.host.id || post.id,
+                    url: videoUrl,
+                    title: post.title,
+                    datetime: post.datetime,
+                    type: 'video/mp4',
+                    original: post
+                }]
+            };
+
+            if(res.status === 200) {
+                item.items[0].mux = [audioUrl];
+            }
+
+            return item;
+        });
+    });
+};
+
+module.exports = redditVideo;

src/save/mux.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const ffmpeg = require('fluent-ffmpeg');
+const fs = require('fs-extra');
+
+function mux(sources, target, item) {
+    return new Promise((resolve, reject) => {
+        return sources.reduce((acc, source) => {
+            return acc.input(source);
+        }, ffmpeg()).videoCodec('copy').audioCodec('copy').on('start', cmd => {
+            console.log('\x1b[36m%s\x1b[0m', `Muxing ${sources.length} streams to '${target}'`);
+        }).on('end', (stdout) => {
+            console.log('\x1b[32m%s\x1b[0m', `Muxed and saved '${target}'`);
+
+            resolve(stdout);
+        }).on('error', error => reject).save(target);
+    }).then(() => {
+        return Promise.all(sources.map(source => {
+            return fs.remove(source);
+        })).then(() => {
+            console.log('\x1b[36m%s\x1b[0m', `Cleaned up temporary files for '${target}'`);
+        });
+    });
+};
+
+module.exports = mux;

src/save/save.js

@@ -2,22 +2,32 @@
 
 const fs = require('fs-extra');
 const path = require('path');
+const ffmpeg = require('fluent-ffmpeg');
+
+function save(filepath, item) {
+    const pathComponents = path.parse(filepath);
 
-function save(filepath, stream) {
     return Promise.resolve().then(() => {
-        return fs.ensureDir(path.dirname(filepath));
+        return fs.ensureDir(path.dirname(pathComponents.dir));
     }).then(() => {
-        const file = fs.createWriteStream(filepath);
+        return Promise.all(item.streams.map((stream, index) => {
+            const target = item.streams.length > 1 ? path.join(pathComponents.dir, `${pathComponents.name}-${index}${pathComponents.ext}`) : filepath;
+            const file = fs.createWriteStream(target);
 
-        return new Promise((resolve, reject) => {
+            return new Promise((resolve, reject) => {
-            stream.pipe(file).on('error', error => {
+                stream.pipe(file).on('error', error => {
-                reject(error);
+                    reject(error);
-            }).on('finish', () => {
+                }).on('finish', () => {
-                console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
+                    if(item.mux) {
+                        console.log(`Temporarily saved '${target}', queued for muxing`);
+                    } else {
+                        console.log('\x1b[32m%s\x1b[0m', `Saved '${target}'`);
+                    }
 
-                resolve(filepath);
+                    resolve(target);
+                });
             });
-        });
+        }));
     });
 };