Added support for reddit preview fallback.

README.md

@@ -98,7 +98,12 @@ Many reddit users have a 'subreddit' of their own in the form of a profile (not
 * `$itemDescription`: The description of the individual image or video
 * `$itemDate`: The submission date of the individual image or video, formatted by the `dateFormat` configuration described below
 * `$itemIndex`: The index of the individual image or video in an album, offset by the `indexOffset` configuration described below
-* `$extracted` (boolean): When extracting single album items is enabled and the item has been extracted, this variable will display the value of `extractedLabel` as described below in case the item was the only item in an album
+* `$extracted` (boolean): Whether the item has been extracted as the only item in an album
+* `$preview` (boolean): Whether the image has been downloaded as a reddit preview because it was unavailable on the original host
+* `$ext`: The extension of the medium. Must typically be included, but may be omitted for self (text) posts on Unix systems
+
+##### `booleans`
+Some variables are booleans and indicate whether or not a property applies. When you use a boolean variable, you must configure a string of text that is only inserted in place of a boolean variable when the variable is true.
 * `$ext`: The extension of the medium. Must typically be included, but may be omitted for self (text) posts on Unix systems
 
 ##### `booleans`

config/default.js

@@ -1,12 +1,12 @@
 module.exports = {
     library: {
         base: 'output/$user/',
-        image: '$base$postDate - $itemId - $postTitle$ext',
+        image: '$base$postDate - $preview$itemId - $postTitle$ext',
-        video: '$base$postDate - $itemId - $postTitle$ext',
+        video: '$base$postDate - $preview$itemId - $postTitle$ext',
-        text: '$base$postDate - $postId - $postTitle',
+        text: '$base$postDate - $preview$postId - $postTitle',
         album: {
-            image: '$base$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
+            image: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext',
-            video: '$base$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext',
+            video: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext',
             extractSingleItem: true
         },
         profile: {
@@ -16,6 +16,7 @@ module.exports = {
         },
         booleans: {
             extracted: 'extracted-',
+            preview: 'preview-',
             verified: '✔',
             verifiedEmail: '✉',
             gold: '★',
@@ -35,6 +36,7 @@ module.exports = {
         avoidDuplicates: true,
         archives: {
             search: false,
+            preview: true,
             reddit: ['ip'],
             reupload: []
         }

src/fetch/info.js

@@ -1,27 +1,29 @@
 'use strict';
 
+const util = require('util');
+const config = require('config');
 const methods = require('../methods/methods.js');
 
 function attachContentInfo(posts) {
     return Promise.all(posts.reduce((acc, post) => {
         if(post.host && methods[post.host.method]) {
-            acc = acc.concat(methods[post.host.method](post).then(content => {
+            acc = acc.concat(methods[post.host.method](post).then(content => Object.assign(post, {content})).catch(error => {
-                post.content = content;
-
-                return post;
-            }).catch(error => {
                 console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);
 
+                if(config.fetch.archives.preview && post.preview) {
+                    console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
+
+                    return methods.redditPreview(post).then(content => Object.assign(post, {content}));
+                }
+
                 return null;
             }));
         } else {
-            console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.id} - ${post.title} - ${post.url}`);
+            console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.url}' (${post.permalink})`);
         }
 
         return acc;
-    }, [])).then(posts => posts.filter(post => {
+    }, [])).then(posts => posts.filter(post => post));
-        return post;
-    }));
 };
 
 module.exports = attachContentInfo;

src/interpolate.js

@@ -64,6 +64,7 @@ function interpolate(pattern, user, post, item) {
             $itemDate: dateFns.format(item.datetime, dateFormat),
             $itemIndex: item.index + config.library.indexOffset,
             $extracted: item.extracted ? config.library.booleans.extracted : '',
+            $preview: item.preview ? config.library.booleans.preview : '',
             $ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname))
         });
     }

src/methods/imgurAlbum.js

@@ -11,7 +11,7 @@ function imgurAlbum(post) {
         }
     }).then(res => res.json()).then(res => {
         if(res.status !== 200) {
-            throw new Error(`Could not fetch info for imgur album '${post.host.id}': ${res.data.error}`);
+            throw new Error(`Could not fetch info for imgur album '${post.host.id}': '${res.data.error}'`);
         }
 
         const extract = config.library.album.extractSingleItem && res.data.images.length === 1;

src/methods/imgurImage.js

@@ -11,7 +11,7 @@ function imgurImage(post) {
         }
     }).then(res => res.json()).then(res => {
         if(res.status !== 200) {
-            throw new Error(`Could not fetch info for imgur image '${post.host.id}': ${res.data.error}`);
+            throw new Error(`Could not fetch info for imgur image '${post.host.id}': '${res.data.error}'`);
         }
 
         return {

src/methods/methods.js

@@ -3,6 +3,7 @@
 const self = require('./self.js');
 const redditImage = require('./redditImage.js');
 const redditVideo = require('./redditVideo.js');
+const redditPreview = require('./redditPreview.js');
 const imgurImage = require('./imgurImage.js');
 const imgurAlbum = require('./imgurAlbum.js');
 const gfycat = require('./gfycat.js');
@@ -12,6 +13,7 @@ module.exports = {
     self,
     redditImage,
     redditVideo,
+    redditPreview,
     imgurImage,
     imgurAlbum,
     gfycat,

src/methods/redditPreview.js

@@ -0,0 +1,40 @@
+ 'use strict';
+
+const util = require('util');
+const config = require('config');
+const path = require('path');
+const fetch = require('node-fetch');
+const urlPattern = require('url-pattern');
+
+const extensions = require('../extensions.json');
+
+function reverseLookup(extension) {
+    return Object.keys(extensions).find(mime => extensions[mime] === extension);
+};
+
+function redditPreview(post) {
+    return Promise.resolve({
+        album: post.preview.length > 1 ? {
+            id: post.host.id || post.id,
+            url: post.url,
+            title: post.title,
+            datetime: post.datetime,
+            original: post
+        } : null,
+        items: post.preview.map(image => {
+            const urlComponents = new urlPattern('http(s)\\://i.redditmedia.com/:id(.:ext)(?*)').match(image.url);
+
+            return {
+                id: post.host.id || post.id,
+                url: image.url,
+                title: post.title,
+                datetime: post.datetime,
+                type: reverseLookup(`.${urlComponents.ext}`),
+                preview: true,
+                original: post
+            };
+        })
+    });
+};
+
+module.exports = redditPreview;