Added support for muxing streams, specifically for reddit-hosted videos (now also supported).

2018-04-26 03:26:39 +02:00 · 2018-04-26 03:26:39 +02:00 · e752667be7
parent d4c842933b
commit e752667be7
11 changed files with 226 additions and 33 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -35,6 +35,14 @@
      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
    },
+    "async": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz",
+      "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==",
+      "requires": {
+        "lodash": "4.17.5"
+      }
+    },
    "asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -209,6 +217,15 @@
      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
    },
+    "define-properties": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
+      "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
+      "requires": {
+        "foreach": "2.0.5",
+        "object-keys": "1.0.11"
+      }
+    },
    "delayed-stream": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -276,6 +293,28 @@
      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
    },
+    "es-abstract": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz",
+      "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==",
+      "requires": {
+        "es-to-primitive": "1.1.1",
+        "function-bind": "1.1.1",
+        "has": "1.0.1",
+        "is-callable": "1.1.3",
+        "is-regex": "1.0.4"
+      }
+    },
+    "es-to-primitive": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
+      "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
+      "requires": {
+        "is-callable": "1.1.3",
+        "is-date-object": "1.0.1",
+        "is-symbol": "1.0.1"
+      }
+    },
    "execa": {
      "version": "0.7.0",
      "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@ -330,6 +369,20 @@
        "locate-path": "2.0.0"
      }
    },
+    "fluent-ffmpeg": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+      "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
+      "requires": {
+        "async": "2.6.0",
+        "which": "1.3.0"
+      }
+    },
+    "foreach": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
+      "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
+    },
    "forever-agent": {
      "version": "0.6.1",
      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@ -355,6 +408,11 @@
        "universalify": "0.1.1"
      }
    },
+    "function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+    },
    "get-caller-file": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz",
@ -397,6 +455,14 @@
      "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.0.tgz",
      "integrity": "sha512-0kZ1XcoelFOLEjEtvWAZyq/1S55eDSieWEJwme311MNVNcRpvjlr2zA66kBV6WAB8C1XI1p1cXCnFPqd1BxlPg=="
    },
+    "has": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz",
+      "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=",
+      "requires": {
+        "function-bind": "1.1.1"
+      }
+    },
    "hawk": {
      "version": "6.0.2",
      "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
@ -446,16 +512,39 @@
      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY="
    },
+    "is-callable": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz",
+      "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
+    },
+    "is-date-object": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
+      "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
+    },
    "is-fullwidth-code-point": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
      "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
    },
+    "is-regex": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
+      "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
+      "requires": {
+        "has": "1.0.1"
+      }
+    },
    "is-stream": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
    },
+    "is-symbol": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
+      "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI="
+    },
    "is-typedarray": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@ -624,6 +713,11 @@
      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
      "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
    },
+    "object-keys": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz",
+      "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0="
+    },
    "options": {
      "version": "0.0.6",
      "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz",
@ -711,6 +805,16 @@
        "harmony-reflect": "1.6.0"
      }
    },
+    "promise.prototype.finally": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.0.tgz",
+      "integrity": "sha512-7p/K2f6dI+dM8yjRQEGrTQs5hTQixUAdOGpMEA3+pVxpX5oHKRSKAXyLw9Q9HUWDTdwtoo39dSHGQtN90HcEwQ==",
+      "requires": {
+        "define-properties": "1.1.2",
+        "es-abstract": "1.11.0",
+        "function-bind": "1.1.1"
+      }
+    },
    "pseudomap": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
--- a/package.json
+++ b/package.json
@ -26,9 +26,11 @@
    "config": "^1.30.0",
    "date-fns": "^1.29.0",
    "dist-exiftool": "^10.53.0",
+    "fluent-ffmpeg": "^2.1.2",
    "fs-extra": "^5.0.0",
    "node-exiftool": "^2.3.0",
    "node-fetch": "^2.1.2",
+    "promise.prototype.finally": "^3.1.0",
    "snoowrap": "^1.15.2",
    "url-pattern": "^1.0.3",
    "yargs": "^11.0.0"
--- a/src/app.js
+++ b/src/app.js
@ -5,6 +5,7 @@ const util = require('util');
 const fs = require('fs-extra');
 const yargs = require('yargs').argv;
 const snoowrap = require('snoowrap');
+const promiseFinally = require('promise.prototype.finally');

 const reddit = new snoowrap(config.reddit.api);

@ -20,6 +21,8 @@ const fetchContent = require('./fetch/content.js');
 const save = require('./save/save.js');
 const saveProfileDetails = require('./save/profileDetails.js');

+promiseFinally.shim();
+
 const limit = yargs.limit || config.fetch.limit;

 if(!yargs.user && typeof yargs.users !== 'string') {
--- a/src/curate/submissions.js
+++ b/src/curate/submissions.js
@ -25,7 +25,7 @@ function curateSubmissions(submissions) {
            title: submission.title,
            text: submission.selftext,
            user: submission.author.name,
-            permalink: submission.permalink,
+            permalink: 'https://reddit.com' + submission.permalink,
            url: submission.url,
            datetime: new Date(submission.created_utc * 1000),
            subreddit: submission.subreddit.display_name,
--- a/src/dissectLink.js
+++ b/src/dissectLink.js
@ -7,13 +7,17 @@ const hosts = [{
    label: 'self',
    pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/')
 }, {
-    method: 'reddit',
+    method: 'redditImage',
    label: 'reddit',
    pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext')
 }, {
-    method: 'reddit',
+    method: 'redditImage',
    label: 'reddit',
-    pattern: new urlPattern('https\\://i.reddituploads.com/:id(?*)')
+    pattern: new urlPattern('http(s)\\://i.reddituploads.com/:id(?*)')
+}, {
+    method: 'redditVideo',
+    label: 'reddit',
+    pattern: new urlPattern('http(s)\\://v.redd.it/:id')
 }, {
    method: 'imgurImage',
    label: 'imgur',
--- a/src/fetch/content.js
+++ b/src/fetch/content.js
@ -9,6 +9,7 @@ const interpolate = require('../interpolate.js');
 const save = require('../save/save.js');
 const textToStream = require('../save/textToStream.js');
 const saveMeta = require('../save/meta.js');
+const mux = require('../save/mux.js');

 const exiftool = require('node-exiftool');
 const exiftoolBin = require('dist-exiftool');
@ -20,19 +21,20 @@ module.exports = function(posts, user) {
        return ep.open();
    }).then(() => {
        return Promise.all(posts.map(post => {
-            return Promise.resolve().then(() => {
-                return Promise.all(post.content.items.map((item, index) => {
-                    item.index = index;
+            return Promise.all(post.content.items.map((item, index) => {
+                item.index = index;

-                    if(item.self) {
-                        return Object.assign({}, item, {stream: textToStream(item.text)});
-                    }
+                if(item.self) {
+                    return Object.assign({}, item, {stream: textToStream(item.text)});
+                }

-                    return fetchItem(item.url, 0).then(stream => {
-                        return Object.assign({}, item, {stream});
-                    });
-                }));
-            }).then(items => {
+                // some videos are delivered with separate audio and are fetched separately to be muxed later
+                const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
+
+                return Promise.all(sources.map(source => {
+                    return fetchItem(source, 0);
+                })).then(streams => Object.assign({}, item, {streams}));
+            })).then(items => {
                return Promise.all(items.map(item => {
                    const type = item.type.split('/')[0];
                    const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item);
@ -40,7 +42,11 @@ module.exports = function(posts, user) {
                    return Promise.resolve().then(() => {
                        return fs.ensureDir(path.dirname(filepath));
                    }).then(() => {
-                        return save(filepath, item.stream);
+                        return save(filepath, item);
+                    }).then(sourcePaths => {
+                        if(item.mux) {
+                            return mux(sourcePaths, filepath, item);
+                        }
                    }).then(() => {
                        const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
                            const interpolatedValue = interpolate(value, user, post, item);
@ -59,9 +65,7 @@ module.exports = function(posts, user) {
                }));
            });
        }));
-    }).then(() => {
-        return ep.close();
-    }).catch(error => {
+    }).finally(() => {
        return ep.close();
    });
 };
--- a/src/methods/methods.js
+++ b/src/methods/methods.js
@ -1,7 +1,8 @@
 'use strict';

 const self = require('./self.js');
-const reddit = require('./reddit.js');
+const redditImage = require('./redditImage.js');
+const redditVideo = require('./redditVideo.js');
 const imgurImage = require('./imgurImage.js');
 const imgurAlbum = require('./imgurAlbum.js');
 const gfycat = require('./gfycat.js');
@ -9,7 +10,8 @@ const eroshare = require('./eroshare.js');

 module.exports = {
    self: self,
-    reddit: reddit,
+    redditImage: redditImage,
+    redditVideo: redditVideo,
    imgurImage: imgurImage,
    imgurAlbum: imgurAlbum,
    gfycat: gfycat,
--- a/src/methods/redditImage.js
+++ b/src/methods/redditImage.js
@ -4,7 +4,7 @@ const util = require('util');
 const config = require('config');
 const fetch = require('node-fetch');

-function reddit(post) {
+function redditImage(post) {
    return Promise.resolve({
        album: null,
        items: [{
@ -18,4 +18,4 @@ function reddit(post) {
    });
 };

-module.exports = reddit;
+module.exports = redditImage;
--- a/src/methods/redditVideo.js
+++ b/src/methods/redditVideo.js
@ -0,0 +1,38 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+const fs = require('fs-extra');
+
+function redditVideo(post) {
+    return fetch(`${post.permalink}.json`).then(res => res.json()).then(res => {
+        return res[0].data.children[0].data.media.reddit_video.fallback_url;
+    }).then(videoUrl => {
+        const audioUrl = videoUrl.split('/').slice(0, -1).join('/') + '/audio';
+
+        return fetch(audioUrl, {
+            method: 'HEAD'
+        }).then(res => {
+            const item = {
+                album: null,
+                items: [{
+                    id: post.host.id || post.id,
+                    url: videoUrl,
+                    title: post.title,
+                    datetime: post.datetime,
+                    type: 'video/mp4',
+                    original: post
+                }]
+            };
+
+            if(res.status === 200) {
+                item.items[0].mux = [audioUrl];
+            }
+
+            return item;
+        });
+    });
+};
+
+module.exports = redditVideo;
--- a/src/save/mux.js
+++ b/src/save/mux.js
@ -0,0 +1,26 @@
+'use strict';
+
+const ffmpeg = require('fluent-ffmpeg');
+const fs = require('fs-extra');
+
+function mux(sources, target, item) {
+    return new Promise((resolve, reject) => {
+        return sources.reduce((acc, source) => {
+            return acc.input(source);
+        }, ffmpeg()).videoCodec('copy').audioCodec('copy').on('start', cmd => {
+            console.log('\x1b[36m%s\x1b[0m', `Muxing ${sources.length} streams to '${target}'`);
+        }).on('end', (stdout) => {
+            console.log('\x1b[32m%s\x1b[0m', `Muxed and saved '${target}'`);
+
+            resolve(stdout);
+        }).on('error', error => reject).save(target);
+    }).then(() => {
+        return Promise.all(sources.map(source => {
+            return fs.remove(source);
+        })).then(() => {
+            console.log('\x1b[36m%s\x1b[0m', `Cleaned up temporary files for '${target}'`);
+        });
+    });
+};
+
+module.exports = mux;
--- a/src/save/save.js
+++ b/src/save/save.js
@ -2,22 +2,32 @@

 const fs = require('fs-extra');
 const path = require('path');
+const ffmpeg = require('fluent-ffmpeg');
+
+function save(filepath, item) {
+    const pathComponents = path.parse(filepath);

-function save(filepath, stream) {
    return Promise.resolve().then(() => {
-        return fs.ensureDir(path.dirname(filepath));
+        return fs.ensureDir(path.dirname(pathComponents.dir));
    }).then(() => {
-        const file = fs.createWriteStream(filepath);
+        return Promise.all(item.streams.map((stream, index) => {
+            const target = item.streams.length > 1 ? path.join(pathComponents.dir, `${pathComponents.name}-${index}${pathComponents.ext}`) : filepath;
+            const file = fs.createWriteStream(target);

-        return new Promise((resolve, reject) => {
-            stream.pipe(file).on('error', error => {
-                reject(error);
-            }).on('finish', () => {
-                console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`);
+            return new Promise((resolve, reject) => {
+                stream.pipe(file).on('error', error => {
+                    reject(error);
+                }).on('finish', () => {
+                    if(item.mux) {
+                        console.log(`Temporarily saved '${target}', queued for muxing`);
+                    } else {
+                        console.log('\x1b[32m%s\x1b[0m', `Saved '${target}'`);
+                    }

-                resolve(filepath);
+                    resolve(target);
+                });
            });
-        });
+        }));
    });
 };