Added support for i.redd.it and self posts. Moved pattern interpolation out of fetch module.

README.md

@@ -28,7 +28,7 @@ Path patterns dictate where and how a file will be saved. Various variables and
 * `$itemDescription`: The description of the individual image or video
 * `$itemDate`: The submission date of the individual image or video, formatted by the `dateformat` configuration described below
 * `$itemIndex`: The index of the individual image or video in an album, offset by the `indexOffset` configuration described below
-* `$ext`: The extension of the file. Must practically always be included.
+* `$ext`: The extension of the file. Must typically be included, but may be omitted for self (text) posts on Unix systems
 
 ### Date format
 Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defaults to `YYYYMMDD`. See [this documentation](https://date-fns.org/v1.29.0/docs/format) for an overview of all available tokens.
@@ -37,4 +37,4 @@ Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defa
 Arrays start at 0, but as to not tire myself out debating the matter, you may offset it my any numerical value you like. Affects the `$itemIndex` variable for album items.
 
 ### Slash substitute
-The patterns are Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute.
+The patterns represent Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute.

app.js

@@ -2,7 +2,6 @@
 
 const config = require('config');
 const util = require('util');
-const note = require('note-log');
 const yargs = require('yargs').argv;
 const snoowrap = require('snoowrap');
 const methods = require('./methods/methods.js');
@@ -12,13 +11,13 @@ const fetchContent = require('./fetchContent.js');
 const reddit = new snoowrap(config.reddit);
 
 reddit.getUser(yargs.user).getSubmissions({
-    sort: 'top',
     limit: Infinity
 }).then(submissions => {
     const curatedPosts = submissions.map(submission => {
         return {
             id: submission.id,
             title: submission.title,
+            text: submission.selftext,
             user: submission.author.name,
             permalink: submission.permalink,
             url: submission.url,
@@ -36,11 +35,11 @@ reddit.getUser(yargs.user).getSubmissions({
                 return post;
             }));
         } else {
-            note('fetch', 1, `"${post.title}": '${post.url}' not supported :(`);
+            console.log('\x1b[33m%s\x1b[0m', `"${post.title}": '${post.url}' not supported :(`);
         }
 
         return acc;
     }, []));
 }).then(posts => fetchContent(posts)).catch(error => {
-    note(error);
+    console.error(error);
 });

dissectLink.js

@@ -3,11 +3,17 @@
 const urlPattern = require('url-pattern');
 
 const hosts = [{
+    method: 'self',
+    pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/')
+}, {
+    method: 'reddit',
+    pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext')
+}, {
     method: 'imgurImage',
     pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
 }, {
     method: 'imgurAlbum',
-    pattern: new urlPattern('http(s)\\://imgur.com/:type/:id')
+    pattern: new urlPattern('http(s)\\://(m.)imgur.com/:type/:id')
 }, {
     method: 'gfycat',
     pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)')

fetchContent.js

@@ -1,70 +1,37 @@
 'use strict';
 
+const Readable = require('stream').Readable;
 const fs = require('fs-extra');
 const path = require('path');
 const config = require('config');
 const fetch = require('node-fetch');
-const dateFns = require('date-fns');
+const interpolate = require('./interpolate.js');
 
-const extensions = {
+function saveItemToDisk(stream, filepath) {
-    'image/jpeg': '.jpg',
+    const file = fs.createWriteStream(filepath);
-    'image/gif': '.gif',
-    'video/mp4': '.mp4',
-    'video/webm': '.webm'
-};
-
-function interpolate(path, post, item, index) {
-    const dateFormat = config.patterns.dateformat || 'YYYYMMDD';
-
-    const vars = {
-        $postId: post.id,
-        $postTitle: post.title,
-        $postUser: post.user,
-        $postDate: dateFns.format(post.datetime, dateFormat)
-    };
-
-    if(post.content.album) {
-        Object.assign(vars, {
-            $albumId: post.content.album.id,
-            $albumTitle: post.content.album.title,
-            $albumDescription: post.content.album.description,
-            $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
-        });
-    }
-
-    if(item) {
-        Object.assign(vars, {
-            $itemId: item.id,
-            $itemTitle: item.title,
-            $itemDescription: item.description,
-            $itemDate: dateFns.format(item.datetime, dateFormat),
-            $itemIndex: index + config.patterns.indexOffset,
-            $ext: extensions[item.type]
-        });
-    }
-
-    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
-        // strip slashes for filesystem compatability
-        value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
-
-        return acc.replace(key, value);
-    }, path);
-};
-
-function saveItemToDisk(buffer, item, index, filename, post) {
-    const stream = fs.createWriteStream(filename);
 
     return new Promise((resolve, reject) => {
-        buffer.body.pipe(stream).on('error', error => {
+        stream.pipe(file).on('error', error => {
             reject(error);
         }).on('finish', () => {
-            console.log(`Saved '${filename}'`);
+            console.log(`Saved '${filepath}'`);
 
-            resolve(filename);
+            resolve(filepath);
         });
     });
 };
 
+function textPostToStream(item) {
+    const stream = new Readable();
+
+    stream.push(item.text);
+    stream.push(null);
+
+    return Object.assign(item, {
+        stream: stream
+    });
+};
+
 function fetchItem(item, index, post, attempt) {
     function retry(error) {
         console.log(error);
@@ -81,7 +48,9 @@ function fetchItem(item, index, post, attempt) {
     }).then(res => {
         console.log(`Fetched '${item.url}'`);
 
-        return res;
+        return Object.assign({}, item, {
+            stream: res.body
+        });
     }).catch(retry);
 };
 
@@ -89,7 +58,11 @@ module.exports = function(posts) {
     return Promise.all(posts.map(post => {
         return Promise.resolve().then(() => {
             return Promise.all(post.content.items.map((item, index) => {
-                return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
+                if(item.self) {
+                    return textPostToStream(item);
+                }
+
+                return fetchItem(item, index, post, 0);
             }));
         }).then(items => {
             return Promise.all(items.map((item, index) => {
@@ -99,7 +72,7 @@ module.exports = function(posts) {
                 return Promise.resolve().then(() => {
                     return fs.ensureDir(path.dirname(filepath));
                 }).then(() => {
-                    return saveItemToDisk(item.buffer, item, index, filepath, post);
+                    return saveItemToDisk(item.stream, filepath)
                 });
             }));
         });

interpolate.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const config = require('config');
+const dateFns = require('date-fns');
+
+const extensions = {
+    'image/jpeg': '.jpg',
+    'image/gif': '.gif',
+    'video/mp4': '.mp4',
+    'video/webm': '.webm'
+};
+
+function interpolate(path, post, item, index) {
+    const dateFormat = config.patterns.dateformat || 'YYYYMMDD';
+
+    const vars = {
+        $postId: post.id,
+        $postTitle: post.title,
+        $postUser: post.user,
+        $postDate: dateFns.format(post.datetime, dateFormat)
+    };
+
+    if(post.content.album) {
+        Object.assign(vars, {
+            $albumId: post.content.album.id,
+            $albumTitle: post.content.album.title,
+            $albumDescription: post.content.album.description,
+            $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
+        });
+    }
+
+    if(item) {
+        Object.assign(vars, {
+            $itemId: item.id,
+            $itemTitle: item.title,
+            $itemDescription: item.description,
+            $itemDate: dateFns.format(item.datetime, dateFormat),
+            $itemIndex: index + config.patterns.indexOffset,
+            $ext: extensions[item.type]
+        });
+    }
+
+    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
+        // strip slashes for filesystem compatability
+        value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
+
+        return acc.replace(key, value);
+    }, path);
+};
+
+module.exports = interpolate;

methods/gfycat.js

@@ -1,11 +1,10 @@
  'use strict';
 
-const note = require('note-log');
 const util = require('util');
 const config = require('config');
 const fetch = require('node-fetch');
 
-function imgurImage(post) {
+function gfycat(post) {
     return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, {
         headers: {
             'Authorization': `Bearer ${config.methods.gfycat.key}`
@@ -24,8 +23,8 @@ function imgurImage(post) {
             }]
         };
     }).catch(error => {
-        note(error);
+        console.error(error);
     });
 };
 
-module.exports = imgurImage;
+module.exports = gfycat;

methods/imgurAlbum.js

@@ -1,6 +1,5 @@
  'use strict';
 
-const note = require('note-log');
 const util = require('util');
 const config = require('config');
 const fetch = require('node-fetch');
@@ -31,7 +30,7 @@ function imgurAlbum(post) {
             }))
         };
     }).catch(error => {
-        note(error);
+        console.error(error);
     });
 };
 

methods/imgurImage.js

@@ -1,6 +1,5 @@
  'use strict';
 
-const note = require('note-log');
 const util = require('util');
 const config = require('config');
 const fetch = require('node-fetch');
@@ -24,7 +23,7 @@ function imgurImage(post) {
             }]
         };
     }).catch(error => {
-        note(error);
+        console.error(error);
     });
 };
 

methods/methods.js

@@ -1,10 +1,14 @@
 'use strict';
 
+const self = require('./self.js');
+const reddit = require('./reddit.js');
 const imgurImage = require('./imgurImage.js');
 const imgurAlbum = require('./imgurAlbum.js');
-const gfycat= require('./gfycat.js');
+const gfycat = require('./gfycat.js');
 
 module.exports = {
+    self: self,
+    reddit: reddit,
     imgurImage: imgurImage,
     imgurAlbum: imgurAlbum,
     gfycat: gfycat

methods/reddit.js

@@ -0,0 +1,21 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function reddit(post) {
+    return Promise.resolve({
+        album: null,
+        items: [{
+            id: post.id,
+            url: post.url,
+            title: post.title,
+            datetime: post.datetime,
+            type: 'image/jpeg',
+            original: post
+        }]
+    });
+};
+
+module.exports = reddit;

methods/self.js

@@ -0,0 +1,23 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function self(post) {
+    return Promise.resolve({
+        album: null,
+        items: [{
+            id: post.id,
+            url: post.url,
+            title: post.title,
+            text: post.text,
+            datetime: post.datetime,
+            type: 'text/plain',
+            self: true,
+            original: post
+        }]
+    });
+};
+
+module.exports = self;

package-lock.json

@@ -77,26 +77,6 @@
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
-    "cli-color": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.2.0.tgz",
-      "integrity": "sha1-OlrnT9drYmevZm5p4q+70B3vNNE=",
-      "requires": {
-        "ansi-regex": "2.1.1",
-        "d": "1.0.0",
-        "es5-ext": "0.10.42",
-        "es6-iterator": "2.0.3",
-        "memoizee": "0.4.12",
-        "timers-ext": "0.1.5"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-          "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
-        }
-      }
-    },
     "cliui": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz",
@@ -167,14 +147,6 @@
         }
       }
     },
-    "d": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz",
-      "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=",
-      "requires": {
-        "es5-ext": "0.10.42"
-      }
-    },
     "dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -207,55 +179,6 @@
         "jsbn": "0.1.1"
       }
     },
-    "es5-ext": {
-      "version": "0.10.42",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.42.tgz",
-      "integrity": "sha512-AJxO1rmPe1bDEfSR6TJ/FgMFYuTBhR5R57KW58iCkYACMyFbrkqVyzXSurYoScDGvgyMpk7uRF/lPUPPTmsRSA==",
-      "requires": {
-        "es6-iterator": "2.0.3",
-        "es6-symbol": "3.1.1",
-        "next-tick": "1.0.0"
-      }
-    },
-    "es6-iterator": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
-      "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
-      "requires": {
-        "d": "1.0.0",
-        "es5-ext": "0.10.42",
-        "es6-symbol": "3.1.1"
-      }
-    },
-    "es6-symbol": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz",
-      "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=",
-      "requires": {
-        "d": "1.0.0",
-        "es5-ext": "0.10.42"
-      }
-    },
-    "es6-weak-map": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.2.tgz",
-      "integrity": "sha1-XjqzIlH/0VOKH45f+hNXdy+S2W8=",
-      "requires": {
-        "d": "1.0.0",
-        "es5-ext": "0.10.42",
-        "es6-iterator": "2.0.3",
-        "es6-symbol": "3.1.1"
-      }
-    },
-    "event-emitter": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
-      "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=",
-      "requires": {
-        "d": "1.0.0",
-        "es5-ext": "0.10.42"
-      }
-    },
     "execa": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@@ -401,11 +324,6 @@
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
       "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
     },
-    "is-promise": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
-      "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
-    },
     "is-stream": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
@@ -502,14 +420,6 @@
         "yallist": "2.1.2"
       }
     },
-    "lru-queue": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz",
-      "integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=",
-      "requires": {
-        "es5-ext": "0.10.42"
-      }
-    },
     "mem": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
@@ -518,21 +428,6 @@
         "mimic-fn": "1.2.0"
       }
     },
-    "memoizee": {
-      "version": "0.4.12",
-      "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.12.tgz",
-      "integrity": "sha512-sprBu6nwxBWBvBOh5v2jcsGqiGLlL2xr2dLub3vR8dnE8YB17omwtm/0NSHl8jjNbcsJd5GMWJAnTSVe/O0Wfg==",
-      "requires": {
-        "d": "1.0.0",
-        "es5-ext": "0.10.42",
-        "es6-weak-map": "2.0.2",
-        "event-emitter": "0.3.5",
-        "is-promise": "2.1.0",
-        "lru-queue": "0.1.0",
-        "next-tick": "1.0.0",
-        "timers-ext": "0.1.5"
-      }
-    },
     "mime-db": {
       "version": "1.33.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
@@ -551,30 +446,11 @@
       "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
       "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
     },
-    "moment": {
-      "version": "2.22.0",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.0.tgz",
-      "integrity": "sha512-1muXCh8jb1N/gHRbn9VDUBr0GYb8A/aVcHlII9QSB68a50spqEVLIGN6KVmCOnSvJrUhC0edGgKU5ofnGXdYdg=="
-    },
-    "next-tick": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
-      "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw="
-    },
     "node-fetch": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
       "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
     },
-    "note-log": {
-      "version": "2.1.11",
-      "resolved": "https://registry.npmjs.org/note-log/-/note-log-2.1.11.tgz",
-      "integrity": "sha1-DvEbJ2llJ2EfG5NHjaXCjyse7SQ=",
-      "requires": {
-        "cli-color": "1.2.0",
-        "moment": "2.22.0"
-      }
-    },
     "npm-run-path": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
@@ -816,15 +692,6 @@
       "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
       "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
     },
-    "timers-ext": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.5.tgz",
-      "integrity": "sha512-tsEStd7kmACHENhsUPaxb8Jf8/+GZZxyNFQbZD07HQOyooOa6At1rQqjffgvg7n+dxscQa9cjjMdWhJtsP2sxg==",
-      "requires": {
-        "es5-ext": "0.10.42",
-        "next-tick": "1.0.0"
-      }
-    },
     "tough-cookie": {
       "version": "2.3.4",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",

package.json

@@ -26,7 +26,6 @@
     "date-fns": "^1.29.0",
     "fs-extra": "^5.0.0",
     "node-fetch": "^2.1.2",
-    "note-log": "^2.1.11",
     "snoowrap": "^1.15.2",
     "url-pattern": "^1.0.3",
     "yargs": "^11.0.0"