Added file save patterns, fetch retry and gfycat support.

app.js

@@ -9,19 +9,20 @@ const methods = require('./methods/methods.js');
 const dissectLink = require('./dissectLink.js');
 const fetchContent = require('./fetchContent.js');
 
-const reddit = new snoowrap(config.api);
+const reddit = new snoowrap(config.reddit);
-const user = yargs.user;
 
-reddit.getUser(user).getSubmissions({
+reddit.getUser(yargs.user).getSubmissions({
-    sort: 'top'
+    sort: 'top',
+    limit: Infinity
 }).then(submissions => {
     const curatedPosts = submissions.map(submission => {
         return {
             id: submission.id,
             title: submission.title,
+            user: submission.author.name,
             permalink: submission.permalink,
             url: submission.url,
-            datetime: submission.created_utc,
+            datetime: submission.created_utc * 1000,
             subreddit: submission.subreddit.display_name,
             host: dissectLink(submission.url)
         };
@@ -40,6 +41,6 @@ reddit.getUser(user).getSubmissions({
 
         return acc;
     }, []));
-}).then(posts => fetchContent(posts, user)).catch(error => {
+}).then(posts => fetchContent(posts)).catch(error => {
     note(error);
 });

dissectLink.js

@@ -8,6 +8,9 @@ const hosts = [{
 }, {
     method: 'imgurAlbum',
     pattern: new urlPattern('http(s)\\://imgur.com/:type/:id')
+}, {
+    method: 'gfycat',
+    pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)')
 }];
 
 module.exports = function dissectLink(url) {

fetchContent.js

@@ -2,45 +2,105 @@
 
 const fs = require('fs-extra');
 const path = require('path');
+const config = require('config');
 const fetch = require('node-fetch');
+const dateFns = require('date-fns');
 
-function saveToDisk(buffer, item, index, filename, post, user) {
+const extensions = {
+    'image/jpeg': '.jpg',
+    'image/gif': '.gif',
+    'video/mp4': '.mp4',
+    'video/webm': '.webm'
+};
+
+function interpolate(path, post, item, index) {
+    const dateFormat = config.patterns.date || 'YYYYMMDD';
+
+    const vars = {
+        $postId: post.id,
+        $postTitle: post.title,
+        $postUser: post.user,
+        $postDate: dateFns.format(post.datetime, dateFormat)
+    };
+
+    if(post.content.album) {
+        Object.assign(vars, {
+            $albumId: post.content.album.id,
+            $albumTitle: post.content.album.title,
+            $albumDescription: post.content.album.description,
+            $albumDate: dateFns.format(post.content.album.datetime, dateFormat)
+        });
+    }
+
+    if(item) {
+        Object.assign(vars, {
+            $itemId: item.id,
+            $itemTitle: item.title,
+            $itemDescription: item.description,
+            $itemDate: dateFns.format(item.datetime, dateFormat),
+            $itemIndex: index + config.patterns.indexOffset,
+            $ext: extensions[item.type]
+        });
+    }
+
+    return Object.entries(vars).reduce((acc, [key, value], index) =>  {
+        // strip slashes for filesystem compatability
+        value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
+
+        return acc.replace(key, value);
+    }, path);
+};
+
+function saveItemToDisk(buffer, item, index, filename, post) {
     const stream = fs.createWriteStream(filename);
 
     return new Promise((resolve, reject) => {
         buffer.body.pipe(stream).on('error', error => {
             reject(error);
         }).on('finish', () => {
-            console.log(`Fetched and saved ${filename}`);
+            console.log(`Saved '${filename}'`);
 
             resolve(filename);
         });
     });
 };
 
-function fetchItem(item, index, post, user) {
+function fetchItem(item, index, post, attempt) {
+    function retry(error) {
+        console.log(error);
+
+        if(attempt < 3) {
+            console.log('Retrying...');
+
+            return fetchItem(item, index, post, ++attempt);
+        }
+    };
+
     return fetch(item.url).then(res => {
         return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
-    });
+    }).then(res => {
+        console.log(`Fetched '${item.url}'`);
+
+        return res;
+    }).catch(retry);
 };
 
-module.exports = function(posts, user) {
+module.exports = function(posts) {
     return Promise.all(posts.map(post => {
-        const filepath = post.content.album ? `output/${user}/${post.content.album.datetime} - ${post.content.album.id} - ${post.title}/` : `output/${user}/`;
-
         return Promise.resolve().then(() => {
-            if(post.content.album) {
-                return fs.ensureDir(filepath);
-            };
-        }).then(() => {
             return Promise.all(post.content.items.map((item, index) => {
-                return fetchItem(item, index, post, user).then(buffer => Object.assign(item, {buffer}));
+                return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
             }));
         }).then(items => {
             return Promise.all(items.map((item, index) => {
-                const filename = post.content.album ? `${filepath}${index + 1} - ${item.id}.jpg` : `${filepath}${item.datetime} - ${item.id}.jpg`;
+                const type = item.type.split('/')[0];
+                const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item, index) : interpolate(config.patterns[type], post, item, index);
 
-                return saveToDisk(item.buffer, item, index, filename, post, user)
+                return Promise.resolve().then(() => {
+                    return fs.ensureDir(path.dirname(filepath));
+                }).then(() => {
+                    return saveItemToDisk(item.buffer, item, index, filepath, post);
+                });
             }));
         });
     }));

methods/gfycat.js

@@ -0,0 +1,31 @@
+ 'use strict';
+
+const note = require('note-log');
+const util = require('util');
+const config = require('config');
+const fetch = require('node-fetch');
+
+function imgurImage(post) {
+    return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, {
+        headers: {
+            'Authorization': `Bearer ${config.methods.gfycat.key}`
+        }
+    }).then(res => res.json()).then(res => {
+        return {
+            album: null,
+            items: [{
+                id: res.gfyItem.gfyName,
+                url: res.gfyItem.webmUrl,
+                title: res.gfyItem.title,
+                description: res.gfyItem.description,
+                type: 'video/webm',
+                datetime: res.gfyItem.createDate * 1000,
+                original: res.gfyItem
+            }]
+        };
+    }).catch(error => {
+        note(error);
+    });
+};
+
+module.exports = imgurImage;

methods/imgurAlbum.js

@@ -17,7 +17,7 @@ function imgurAlbum(post) {
                 url: res.data.link,
                 title: res.data.title,
                 description: res.data.description,
-                datetime: res.data.datetime,
+                datetime: res.data.datetime * 1000,
                 original: res.data
             },
             items: res.data.images.map(item => ({
@@ -25,7 +25,7 @@ function imgurAlbum(post) {
                 url: item.link,
                 title: item.title,
                 description: item.description,
-                datetime: item.datetime,
+                datetime: item.datetime * 1000,
                 type: item.type,
                 original: item
             }))

methods/imgurImage.js

@@ -19,7 +19,7 @@ function imgurImage(post) {
                 title: res.data.title,
                 description: res.data.description,
                 type: res.data.type,
-                datetime: res.data.datetime,
+                datetime: res.data.datetime * 1000,
                 original: res.data
             }]
         };

methods/methods.js

@@ -2,8 +2,10 @@
 
 const imgurImage = require('./imgurImage.js');
 const imgurAlbum = require('./imgurAlbum.js');
+const gfycat= require('./gfycat.js');
 
 module.exports = {
     imgurImage: imgurImage,
-    imgurAlbum: imgurAlbum
+    imgurAlbum: imgurAlbum,
+    gfycat: gfycat
 };

package-lock.json

@@ -183,6 +183,11 @@
         "assert-plus": "1.0.0"
       }
     },
+    "date-fns": {
+      "version": "1.29.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
+      "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
+    },
     "decamelize": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",

package.json

@@ -23,6 +23,7 @@
   "license": "ISC",
   "dependencies": {
     "config": "^1.30.0",
+    "date-fns": "^1.29.0",
     "fs-extra": "^5.0.0",
     "node-fetch": "^2.1.2",
     "note-log": "^2.1.11",