From 7f00e5b6b28f991af95d92753e94319963bad4ff Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:54 +0200 Subject: [PATCH] Added support for reddit preview fallback. --- README.md | 7 ++++++- config/default.js | 12 ++++++----- src/fetch/info.js | 20 ++++++++++-------- src/interpolate.js | 1 + src/methods/imgurAlbum.js | 2 +- src/methods/imgurImage.js | 2 +- src/methods/methods.js | 2 ++ src/methods/redditPreview.js | 40 ++++++++++++++++++++++++++++++++++++ 8 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 src/methods/redditPreview.js diff --git a/README.md b/README.md index fda8291..30c7848 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,12 @@ Many reddit users have a 'subreddit' of their own in the form of a profile (not * `$itemDescription`: The description of the individual image or video * `$itemDate`: The submission date of the individual image or video, formatted by the `dateFormat` configuration described below * `$itemIndex`: The index of the individual image or video in an album, offset by the `indexOffset` configuration described below -* `$extracted` (boolean): When extracting single album items is enabled and the item has been extracted, this variable will display the value of `extractedLabel` as described below in case the item was the only item in an album +* `$extracted` (boolean): Whether the item has been extracted as the only item in an album +* `$preview` (boolean): Whether the image has been downloaded as a reddit preview because it was unavailable on the original host +* `$ext`: The extension of the medium. Must typically be included, but may be omitted for self (text) posts on Unix systems + +##### `booleans` +Some variables are booleans and indicate whether or not a property applies. When you use a boolean variable, you must configure a string of text that is only inserted in place of a boolean variable when the variable is true. * `$ext`: The extension of the medium. Must typically be included, but may be omitted for self (text) posts on Unix systems ##### `booleans` diff --git a/config/default.js b/config/default.js index 0369960..bb52e77 100644 --- a/config/default.js +++ b/config/default.js @@ -1,12 +1,12 @@ module.exports = { library: { base: 'output/$user/', - image: '$base$postDate - $itemId - $postTitle$ext', - video: '$base$postDate - $itemId - $postTitle$ext', - text: '$base$postDate - $postId - $postTitle', + image: '$base$postDate - $preview$itemId - $postTitle$ext', + video: '$base$postDate - $preview$itemId - $postTitle$ext', + text: '$base$postDate - $preview$postId - $postTitle', album: { - image: '$base$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext', - video: '$base$postDate - $albumId - $postTitle/$itemIndex - $itemId$ext', + image: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext', + video: '$base$postDate - $preview$albumId - $postTitle/$itemIndex - $itemId$ext', extractSingleItem: true }, profile: { @@ -16,6 +16,7 @@ module.exports = { }, booleans: { extracted: 'extracted-', + preview: 'preview-', verified: '✔', verifiedEmail: '✉', gold: '★', @@ -35,6 +36,7 @@ module.exports = { avoidDuplicates: true, archives: { search: false, + preview: true, reddit: ['ip'], reupload: [] } diff --git a/src/fetch/info.js b/src/fetch/info.js index 0019f00..ddfc5b3 100644 --- a/src/fetch/info.js +++ b/src/fetch/info.js @@ -1,27 +1,29 @@ 'use strict'; +const util = require('util'); +const config = require('config'); const methods = require('../methods/methods.js'); function attachContentInfo(posts) { return Promise.all(posts.reduce((acc, post) => { if(post.host && methods[post.host.method]) { - acc = acc.concat(methods[post.host.method](post).then(content => { - post.content = content; - - return post; - }).catch(error => { + acc = acc.concat(methods[post.host.method](post).then(content => Object.assign(post, {content})).catch(error => { console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`); + if(config.fetch.archives.preview && post.preview) { + console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`); + + return methods.redditPreview(post).then(content => Object.assign(post, {content})); + } + return null; })); } else { - console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.id} - ${post.title} - ${post.url}`); + console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.url}' (${post.permalink})`); } return acc; - }, [])).then(posts => posts.filter(post => { - return post; - })); + }, [])).then(posts => posts.filter(post => post)); }; module.exports = attachContentInfo; diff --git a/src/interpolate.js b/src/interpolate.js index 9baa6a7..1d81bb8 100644 --- a/src/interpolate.js +++ b/src/interpolate.js @@ -64,6 +64,7 @@ function interpolate(pattern, user, post, item) { $itemDate: dateFns.format(item.datetime, dateFormat), $itemIndex: item.index + config.library.indexOffset, $extracted: item.extracted ? config.library.booleans.extracted : '', + $preview: item.preview ? config.library.booleans.preview : '', $ext: item.extension || (item.type ? extensions[item.type] : path.extname(url.parse(item.url).pathname)) }); } diff --git a/src/methods/imgurAlbum.js b/src/methods/imgurAlbum.js index 24084e5..2d8b6fa 100644 --- a/src/methods/imgurAlbum.js +++ b/src/methods/imgurAlbum.js @@ -11,7 +11,7 @@ function imgurAlbum(post) { } }).then(res => res.json()).then(res => { if(res.status !== 200) { - throw new Error(`Could not fetch info for imgur album '${post.host.id}': ${res.data.error}`); + throw new Error(`Could not fetch info for imgur album '${post.host.id}': '${res.data.error}'`); } const extract = config.library.album.extractSingleItem && res.data.images.length === 1; diff --git a/src/methods/imgurImage.js b/src/methods/imgurImage.js index 227ddd2..09f646a 100644 --- a/src/methods/imgurImage.js +++ b/src/methods/imgurImage.js @@ -11,7 +11,7 @@ function imgurImage(post) { } }).then(res => res.json()).then(res => { if(res.status !== 200) { - throw new Error(`Could not fetch info for imgur image '${post.host.id}': ${res.data.error}`); + throw new Error(`Could not fetch info for imgur image '${post.host.id}': '${res.data.error}'`); } return { diff --git a/src/methods/methods.js b/src/methods/methods.js index 2db9ac1..9a6bbd0 100644 --- a/src/methods/methods.js +++ b/src/methods/methods.js @@ -3,6 +3,7 @@ const self = require('./self.js'); const redditImage = require('./redditImage.js'); const redditVideo = require('./redditVideo.js'); +const redditPreview = require('./redditPreview.js'); const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); const gfycat = require('./gfycat.js'); @@ -12,6 +13,7 @@ module.exports = { self, redditImage, redditVideo, + redditPreview, imgurImage, imgurAlbum, gfycat, diff --git a/src/methods/redditPreview.js b/src/methods/redditPreview.js new file mode 100644 index 0000000..44a55fb --- /dev/null +++ b/src/methods/redditPreview.js @@ -0,0 +1,40 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const path = require('path'); +const fetch = require('node-fetch'); +const urlPattern = require('url-pattern'); + +const extensions = require('../extensions.json'); + +function reverseLookup(extension) { + return Object.keys(extensions).find(mime => extensions[mime] === extension); +}; + +function redditPreview(post) { + return Promise.resolve({ + album: post.preview.length > 1 ? { + id: post.host.id || post.id, + url: post.url, + title: post.title, + datetime: post.datetime, + original: post + } : null, + items: post.preview.map(image => { + const urlComponents = new urlPattern('http(s)\\://i.redditmedia.com/:id(.:ext)(?*)').match(image.url); + + return { + id: post.host.id || post.id, + url: image.url, + title: post.title, + datetime: post.datetime, + type: reverseLookup(`.${urlComponents.ext}`), + preview: true, + original: post + }; + }) + }); +}; + +module.exports = redditPreview;