diff --git a/package-lock.json b/package-lock.json index edb242f..724845f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,14 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", + "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", + "requires": { + "lodash": "4.17.5" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -209,6 +217,15 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, + "define-properties": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", + "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", + "requires": { + "foreach": "2.0.5", + "object-keys": "1.0.11" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -276,6 +293,28 @@ "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" }, + "es-abstract": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz", + "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==", + "requires": { + "es-to-primitive": "1.1.1", + "function-bind": "1.1.1", + "has": "1.0.1", + "is-callable": "1.1.3", + "is-regex": "1.0.4" + } + }, + "es-to-primitive": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz", + "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=", + "requires": { + "is-callable": "1.1.3", + "is-date-object": "1.0.1", + "is-symbol": "1.0.1" + } + }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -330,6 +369,20 @@ "locate-path": "2.0.0" } }, + "fluent-ffmpeg": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", + "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=", + "requires": { + "async": "2.6.0", + "which": "1.3.0" + } + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -355,6 +408,11 @@ "universalify": "0.1.1" } }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "get-caller-file": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz", @@ -397,6 +455,14 @@ "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.0.tgz", "integrity": "sha512-0kZ1XcoelFOLEjEtvWAZyq/1S55eDSieWEJwme311MNVNcRpvjlr2zA66kBV6WAB8C1XI1p1cXCnFPqd1BxlPg==" }, + "has": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz", + "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=", + "requires": { + "function-bind": "1.1.1" + } + }, "hawk": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", @@ -446,16 +512,39 @@ "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" }, + "is-callable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz", + "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI=" + }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "requires": { + "has": "1.0.1" + } + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, + "is-symbol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz", + "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=" + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -624,6 +713,11 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" }, + "object-keys": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", + "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=" + }, "options": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", @@ -711,6 +805,16 @@ "harmony-reflect": "1.6.0" } }, + "promise.prototype.finally": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.0.tgz", + "integrity": "sha512-7p/K2f6dI+dM8yjRQEGrTQs5hTQixUAdOGpMEA3+pVxpX5oHKRSKAXyLw9Q9HUWDTdwtoo39dSHGQtN90HcEwQ==", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0", + "function-bind": "1.1.1" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", diff --git a/package.json b/package.json index 427e84e..c1a757b 100644 --- a/package.json +++ b/package.json @@ -26,9 +26,11 @@ "config": "^1.30.0", "date-fns": "^1.29.0", "dist-exiftool": "^10.53.0", + "fluent-ffmpeg": "^2.1.2", "fs-extra": "^5.0.0", "node-exiftool": "^2.3.0", "node-fetch": "^2.1.2", + "promise.prototype.finally": "^3.1.0", "snoowrap": "^1.15.2", "url-pattern": "^1.0.3", "yargs": "^11.0.0" diff --git a/src/app.js b/src/app.js index eb5edd1..ab2a52a 100644 --- a/src/app.js +++ b/src/app.js @@ -5,6 +5,7 @@ const util = require('util'); const fs = require('fs-extra'); const yargs = require('yargs').argv; const snoowrap = require('snoowrap'); +const promiseFinally = require('promise.prototype.finally'); const reddit = new snoowrap(config.reddit.api); @@ -20,6 +21,8 @@ const fetchContent = require('./fetch/content.js'); const save = require('./save/save.js'); const saveProfileDetails = require('./save/profileDetails.js'); +promiseFinally.shim(); + const limit = yargs.limit || config.fetch.limit; if(!yargs.user && typeof yargs.users !== 'string') { diff --git a/src/curate/submissions.js b/src/curate/submissions.js index 0455c61..7eea792 100644 --- a/src/curate/submissions.js +++ b/src/curate/submissions.js @@ -25,7 +25,7 @@ function curateSubmissions(submissions) { title: submission.title, text: submission.selftext, user: submission.author.name, - permalink: submission.permalink, + permalink: 'https://reddit.com' + submission.permalink, url: submission.url, datetime: new Date(submission.created_utc * 1000), subreddit: submission.subreddit.display_name, diff --git a/src/dissectLink.js b/src/dissectLink.js index 3cbaeb1..83ab086 100644 --- a/src/dissectLink.js +++ b/src/dissectLink.js @@ -7,13 +7,17 @@ const hosts = [{ label: 'self', pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/') }, { - method: 'reddit', + method: 'redditImage', label: 'reddit', pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext') }, { - method: 'reddit', + method: 'redditImage', label: 'reddit', - pattern: new urlPattern('https\\://i.reddituploads.com/:id(?*)') + pattern: new urlPattern('http(s)\\://i.reddituploads.com/:id(?*)') +}, { + method: 'redditVideo', + label: 'reddit', + pattern: new urlPattern('http(s)\\://v.redd.it/:id') }, { method: 'imgurImage', label: 'imgur', diff --git a/src/fetch/content.js b/src/fetch/content.js index 29c8e56..36f2c6f 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -9,6 +9,7 @@ const interpolate = require('../interpolate.js'); const save = require('../save/save.js'); const textToStream = require('../save/textToStream.js'); const saveMeta = require('../save/meta.js'); +const mux = require('../save/mux.js'); const exiftool = require('node-exiftool'); const exiftoolBin = require('dist-exiftool'); @@ -20,19 +21,20 @@ module.exports = function(posts, user) { return ep.open(); }).then(() => { return Promise.all(posts.map(post => { - return Promise.resolve().then(() => { - return Promise.all(post.content.items.map((item, index) => { - item.index = index; + return Promise.all(post.content.items.map((item, index) => { + item.index = index; - if(item.self) { - return Object.assign({}, item, {stream: textToStream(item.text)}); - } + if(item.self) { + return Object.assign({}, item, {stream: textToStream(item.text)}); + } - return fetchItem(item.url, 0).then(stream => { - return Object.assign({}, item, {stream}); - }); - })); - }).then(items => { + // some videos are delivered with separate audio and are fetched separately to be muxed later + const sources = item.mux ? [item.url].concat(item.mux) : [item.url]; + + return Promise.all(sources.map(source => { + return fetchItem(source, 0); + })).then(streams => Object.assign({}, item, {streams})); + })).then(items => { return Promise.all(items.map(item => { const type = item.type.split('/')[0]; const filepath = post.content.album ? interpolate(config.library.album[type], user, post, item) : interpolate(config.library[type], user, post, item); @@ -40,7 +42,11 @@ module.exports = function(posts, user) { return Promise.resolve().then(() => { return fs.ensureDir(path.dirname(filepath)); }).then(() => { - return save(filepath, item.stream); + return save(filepath, item); + }).then(sourcePaths => { + if(item.mux) { + return mux(sourcePaths, filepath, item); + } }).then(() => { const meta = Object.entries(config.library.meta).reduce((acc, [key, value]) => { const interpolatedValue = interpolate(value, user, post, item); @@ -59,9 +65,7 @@ module.exports = function(posts, user) { })); }); })); - }).then(() => { - return ep.close(); - }).catch(error => { + }).finally(() => { return ep.close(); }); }; diff --git a/src/methods/methods.js b/src/methods/methods.js index aad635a..5b45a67 100644 --- a/src/methods/methods.js +++ b/src/methods/methods.js @@ -1,7 +1,8 @@ 'use strict'; const self = require('./self.js'); -const reddit = require('./reddit.js'); +const redditImage = require('./redditImage.js'); +const redditVideo = require('./redditVideo.js'); const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); const gfycat = require('./gfycat.js'); @@ -9,7 +10,8 @@ const eroshare = require('./eroshare.js'); module.exports = { self: self, - reddit: reddit, + redditImage: redditImage, + redditVideo: redditVideo, imgurImage: imgurImage, imgurAlbum: imgurAlbum, gfycat: gfycat, diff --git a/src/methods/reddit.js b/src/methods/redditImage.js similarity index 87% rename from src/methods/reddit.js rename to src/methods/redditImage.js index 24114f4..791e81e 100644 --- a/src/methods/reddit.js +++ b/src/methods/redditImage.js @@ -4,7 +4,7 @@ const util = require('util'); const config = require('config'); const fetch = require('node-fetch'); -function reddit(post) { +function redditImage(post) { return Promise.resolve({ album: null, items: [{ @@ -18,4 +18,4 @@ function reddit(post) { }); }; -module.exports = reddit; +module.exports = redditImage; diff --git a/src/methods/redditVideo.js b/src/methods/redditVideo.js new file mode 100644 index 0000000..137aabf --- /dev/null +++ b/src/methods/redditVideo.js @@ -0,0 +1,38 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); +const fs = require('fs-extra'); + +function redditVideo(post) { + return fetch(`${post.permalink}.json`).then(res => res.json()).then(res => { + return res[0].data.children[0].data.media.reddit_video.fallback_url; + }).then(videoUrl => { + const audioUrl = videoUrl.split('/').slice(0, -1).join('/') + '/audio'; + + return fetch(audioUrl, { + method: 'HEAD' + }).then(res => { + const item = { + album: null, + items: [{ + id: post.host.id || post.id, + url: videoUrl, + title: post.title, + datetime: post.datetime, + type: 'video/mp4', + original: post + }] + }; + + if(res.status === 200) { + item.items[0].mux = [audioUrl]; + } + + return item; + }); + }); +}; + +module.exports = redditVideo; diff --git a/src/save/mux.js b/src/save/mux.js new file mode 100644 index 0000000..016d539 --- /dev/null +++ b/src/save/mux.js @@ -0,0 +1,26 @@ +'use strict'; + +const ffmpeg = require('fluent-ffmpeg'); +const fs = require('fs-extra'); + +function mux(sources, target, item) { + return new Promise((resolve, reject) => { + return sources.reduce((acc, source) => { + return acc.input(source); + }, ffmpeg()).videoCodec('copy').audioCodec('copy').on('start', cmd => { + console.log('\x1b[36m%s\x1b[0m', `Muxing ${sources.length} streams to '${target}'`); + }).on('end', (stdout) => { + console.log('\x1b[32m%s\x1b[0m', `Muxed and saved '${target}'`); + + resolve(stdout); + }).on('error', error => reject).save(target); + }).then(() => { + return Promise.all(sources.map(source => { + return fs.remove(source); + })).then(() => { + console.log('\x1b[36m%s\x1b[0m', `Cleaned up temporary files for '${target}'`); + }); + }); +}; + +module.exports = mux; diff --git a/src/save/save.js b/src/save/save.js index bb70355..42461f8 100644 --- a/src/save/save.js +++ b/src/save/save.js @@ -2,22 +2,32 @@ const fs = require('fs-extra'); const path = require('path'); +const ffmpeg = require('fluent-ffmpeg'); + +function save(filepath, item) { + const pathComponents = path.parse(filepath); -function save(filepath, stream) { return Promise.resolve().then(() => { - return fs.ensureDir(path.dirname(filepath)); + return fs.ensureDir(path.dirname(pathComponents.dir)); }).then(() => { - const file = fs.createWriteStream(filepath); + return Promise.all(item.streams.map((stream, index) => { + const target = item.streams.length > 1 ? path.join(pathComponents.dir, `${pathComponents.name}-${index}${pathComponents.ext}`) : filepath; + const file = fs.createWriteStream(target); - return new Promise((resolve, reject) => { - stream.pipe(file).on('error', error => { - reject(error); - }).on('finish', () => { - console.log('\x1b[32m%s\x1b[0m', `Saved '${filepath}'`); + return new Promise((resolve, reject) => { + stream.pipe(file).on('error', error => { + reject(error); + }).on('finish', () => { + if(item.mux) { + console.log(`Temporarily saved '${target}', queued for muxing`); + } else { + console.log('\x1b[32m%s\x1b[0m', `Saved '${target}'`); + } - resolve(filepath); + resolve(target); + }); }); - }); + })); }); };