From f5e8906823be3fd3ff7e4492eaf2cc87079882c2 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:53 +0200 Subject: [PATCH] Added support for i.redd.it and self posts. Moved pattern interpolation out of fetch module. --- README.md | 4 +- app.js | 7 +-- dissectLink.js | 8 ++- fetchContent.js | 81 +++++++++---------------- interpolate.js | 51 ++++++++++++++++ methods/gfycat.js | 7 +-- methods/imgurAlbum.js | 3 +- methods/imgurImage.js | 3 +- methods/methods.js | 6 +- methods/reddit.js | 21 +++++++ methods/self.js | 23 ++++++++ package-lock.json | 133 ------------------------------------------ package.json | 1 - 13 files changed, 144 insertions(+), 204 deletions(-) create mode 100644 interpolate.js create mode 100644 methods/reddit.js create mode 100644 methods/self.js diff --git a/README.md b/README.md index 55aa991..c03be79 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Path patterns dictate where and how a file will be saved. Various variables and * `$itemDescription`: The description of the individual image or video * `$itemDate`: The submission date of the individual image or video, formatted by the `dateformat` configuration described below * `$itemIndex`: The index of the individual image or video in an album, offset by the `indexOffset` configuration described below -* `$ext`: The extension of the file. Must practically always be included. +* `$ext`: The extension of the file. Must typically be included, but may be omitted for self (text) posts on Unix systems ### Date format Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defaults to `YYYYMMDD`. See [this documentation](https://date-fns.org/v1.29.0/docs/format) for an overview of all available tokens. @@ -37,4 +37,4 @@ Affects the representation of `$postDate`, `$albumDate` and `$itemDate` and defa Arrays start at 0, but as to not tire myself out debating the matter, you may offset it my any numerical value you like. Affects the `$itemIndex` variable for album items. ### Slash substitute -The patterns are Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute. +The patterns represent Unix file paths, and a `/` therefore indicates a new directory. You may freely use directories in your paths, but titles or descriptions may contain a `/` that is not supposed to create a new directory. All instances of `/` in a variable value will be replaced with the configured slash substitute. diff --git a/app.js b/app.js index 663f0b4..8c24671 100644 --- a/app.js +++ b/app.js @@ -2,7 +2,6 @@ const config = require('config'); const util = require('util'); -const note = require('note-log'); const yargs = require('yargs').argv; const snoowrap = require('snoowrap'); const methods = require('./methods/methods.js'); @@ -12,13 +11,13 @@ const fetchContent = require('./fetchContent.js'); const reddit = new snoowrap(config.reddit); reddit.getUser(yargs.user).getSubmissions({ - sort: 'top', limit: Infinity }).then(submissions => { const curatedPosts = submissions.map(submission => { return { id: submission.id, title: submission.title, + text: submission.selftext, user: submission.author.name, permalink: submission.permalink, url: submission.url, @@ -36,11 +35,11 @@ reddit.getUser(yargs.user).getSubmissions({ return post; })); } else { - note('fetch', 1, `"${post.title}": '${post.url}' not supported :(`); + console.log('\x1b[33m%s\x1b[0m', `"${post.title}": '${post.url}' not supported :(`); } return acc; }, [])); }).then(posts => fetchContent(posts)).catch(error => { - note(error); + console.error(error); }); diff --git a/dissectLink.js b/dissectLink.js index 33603de..322dbbf 100644 --- a/dissectLink.js +++ b/dissectLink.js @@ -3,11 +3,17 @@ const urlPattern = require('url-pattern'); const hosts = [{ + method: 'self', + pattern: new urlPattern('http(s)\\://(www.)reddit.com/r/:subreddit/comments/:id/:uri/') +}, { + method: 'reddit', + pattern: new urlPattern('http(s)\\://i.redd.it/:id.:ext') +}, { method: 'imgurImage', pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)') }, { method: 'imgurAlbum', - pattern: new urlPattern('http(s)\\://imgur.com/:type/:id') + pattern: new urlPattern('http(s)\\://(m.)imgur.com/:type/:id') }, { method: 'gfycat', pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)') diff --git a/fetchContent.js b/fetchContent.js index ce3c542..7b06424 100644 --- a/fetchContent.js +++ b/fetchContent.js @@ -1,70 +1,37 @@ 'use strict'; +const Readable = require('stream').Readable; const fs = require('fs-extra'); const path = require('path'); const config = require('config'); const fetch = require('node-fetch'); -const dateFns = require('date-fns'); +const interpolate = require('./interpolate.js'); -const extensions = { - 'image/jpeg': '.jpg', - 'image/gif': '.gif', - 'video/mp4': '.mp4', - 'video/webm': '.webm' -}; - -function interpolate(path, post, item, index) { - const dateFormat = config.patterns.dateformat || 'YYYYMMDD'; - - const vars = { - $postId: post.id, - $postTitle: post.title, - $postUser: post.user, - $postDate: dateFns.format(post.datetime, dateFormat) - }; - - if(post.content.album) { - Object.assign(vars, { - $albumId: post.content.album.id, - $albumTitle: post.content.album.title, - $albumDescription: post.content.album.description, - $albumDate: dateFns.format(post.content.album.datetime, dateFormat) - }); - } - - if(item) { - Object.assign(vars, { - $itemId: item.id, - $itemTitle: item.title, - $itemDescription: item.description, - $itemDate: dateFns.format(item.datetime, dateFormat), - $itemIndex: index + config.patterns.indexOffset, - $ext: extensions[item.type] - }); - } - - return Object.entries(vars).reduce((acc, [key, value], index) => { - // strip slashes for filesystem compatability - value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute); - - return acc.replace(key, value); - }, path); -}; - -function saveItemToDisk(buffer, item, index, filename, post) { - const stream = fs.createWriteStream(filename); +function saveItemToDisk(stream, filepath) { + const file = fs.createWriteStream(filepath); return new Promise((resolve, reject) => { - buffer.body.pipe(stream).on('error', error => { + stream.pipe(file).on('error', error => { reject(error); }).on('finish', () => { - console.log(`Saved '${filename}'`); + console.log(`Saved '${filepath}'`); - resolve(filename); + resolve(filepath); }); }); }; +function textPostToStream(item) { + const stream = new Readable(); + + stream.push(item.text); + stream.push(null); + + return Object.assign(item, { + stream: stream + }); +}; + function fetchItem(item, index, post, attempt) { function retry(error) { console.log(error); @@ -81,7 +48,9 @@ function fetchItem(item, index, post, attempt) { }).then(res => { console.log(`Fetched '${item.url}'`); - return res; + return Object.assign({}, item, { + stream: res.body + }); }).catch(retry); }; @@ -89,7 +58,11 @@ module.exports = function(posts) { return Promise.all(posts.map(post => { return Promise.resolve().then(() => { return Promise.all(post.content.items.map((item, index) => { - return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer})); + if(item.self) { + return textPostToStream(item); + } + + return fetchItem(item, index, post, 0); })); }).then(items => { return Promise.all(items.map((item, index) => { @@ -99,7 +72,7 @@ module.exports = function(posts) { return Promise.resolve().then(() => { return fs.ensureDir(path.dirname(filepath)); }).then(() => { - return saveItemToDisk(item.buffer, item, index, filepath, post); + return saveItemToDisk(item.stream, filepath) }); })); }); diff --git a/interpolate.js b/interpolate.js new file mode 100644 index 0000000..ef0605d --- /dev/null +++ b/interpolate.js @@ -0,0 +1,51 @@ +'use strict'; + +const config = require('config'); +const dateFns = require('date-fns'); + +const extensions = { + 'image/jpeg': '.jpg', + 'image/gif': '.gif', + 'video/mp4': '.mp4', + 'video/webm': '.webm' +}; + +function interpolate(path, post, item, index) { + const dateFormat = config.patterns.dateformat || 'YYYYMMDD'; + + const vars = { + $postId: post.id, + $postTitle: post.title, + $postUser: post.user, + $postDate: dateFns.format(post.datetime, dateFormat) + }; + + if(post.content.album) { + Object.assign(vars, { + $albumId: post.content.album.id, + $albumTitle: post.content.album.title, + $albumDescription: post.content.album.description, + $albumDate: dateFns.format(post.content.album.datetime, dateFormat) + }); + } + + if(item) { + Object.assign(vars, { + $itemId: item.id, + $itemTitle: item.title, + $itemDescription: item.description, + $itemDate: dateFns.format(item.datetime, dateFormat), + $itemIndex: index + config.patterns.indexOffset, + $ext: extensions[item.type] + }); + } + + return Object.entries(vars).reduce((acc, [key, value], index) => { + // strip slashes for filesystem compatability + value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute); + + return acc.replace(key, value); + }, path); +}; + +module.exports = interpolate; diff --git a/methods/gfycat.js b/methods/gfycat.js index 3cd91df..208235c 100644 --- a/methods/gfycat.js +++ b/methods/gfycat.js @@ -1,11 +1,10 @@ 'use strict'; -const note = require('note-log'); const util = require('util'); const config = require('config'); const fetch = require('node-fetch'); -function imgurImage(post) { +function gfycat(post) { return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, { headers: { 'Authorization': `Bearer ${config.methods.gfycat.key}` @@ -24,8 +23,8 @@ function imgurImage(post) { }] }; }).catch(error => { - note(error); + console.error(error); }); }; -module.exports = imgurImage; +module.exports = gfycat; diff --git a/methods/imgurAlbum.js b/methods/imgurAlbum.js index a3d8e5e..18ddfb6 100644 --- a/methods/imgurAlbum.js +++ b/methods/imgurAlbum.js @@ -1,6 +1,5 @@ 'use strict'; -const note = require('note-log'); const util = require('util'); const config = require('config'); const fetch = require('node-fetch'); @@ -31,7 +30,7 @@ function imgurAlbum(post) { })) }; }).catch(error => { - note(error); + console.error(error); }); }; diff --git a/methods/imgurImage.js b/methods/imgurImage.js index ebdd338..920f8bf 100644 --- a/methods/imgurImage.js +++ b/methods/imgurImage.js @@ -1,6 +1,5 @@ 'use strict'; -const note = require('note-log'); const util = require('util'); const config = require('config'); const fetch = require('node-fetch'); @@ -24,7 +23,7 @@ function imgurImage(post) { }] }; }).catch(error => { - note(error); + console.error(error); }); }; diff --git a/methods/methods.js b/methods/methods.js index 317fe41..ab5aead 100644 --- a/methods/methods.js +++ b/methods/methods.js @@ -1,10 +1,14 @@ 'use strict'; +const self = require('./self.js'); +const reddit = require('./reddit.js'); const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); -const gfycat= require('./gfycat.js'); +const gfycat = require('./gfycat.js'); module.exports = { + self: self, + reddit: reddit, imgurImage: imgurImage, imgurAlbum: imgurAlbum, gfycat: gfycat diff --git a/methods/reddit.js b/methods/reddit.js new file mode 100644 index 0000000..80e4ddb --- /dev/null +++ b/methods/reddit.js @@ -0,0 +1,21 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); + +function reddit(post) { + return Promise.resolve({ + album: null, + items: [{ + id: post.id, + url: post.url, + title: post.title, + datetime: post.datetime, + type: 'image/jpeg', + original: post + }] + }); +}; + +module.exports = reddit; diff --git a/methods/self.js b/methods/self.js new file mode 100644 index 0000000..062454b --- /dev/null +++ b/methods/self.js @@ -0,0 +1,23 @@ + 'use strict'; + +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); + +function self(post) { + return Promise.resolve({ + album: null, + items: [{ + id: post.id, + url: post.url, + title: post.title, + text: post.text, + datetime: post.datetime, + type: 'text/plain', + self: true, + original: post + }] + }); +}; + +module.exports = self; diff --git a/package-lock.json b/package-lock.json index 61f6543..aad6645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -77,26 +77,6 @@ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, - "cli-color": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.2.0.tgz", - "integrity": "sha1-OlrnT9drYmevZm5p4q+70B3vNNE=", - "requires": { - "ansi-regex": "2.1.1", - "d": "1.0.0", - "es5-ext": "0.10.42", - "es6-iterator": "2.0.3", - "memoizee": "0.4.12", - "timers-ext": "0.1.5" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - } - } - }, "cliui": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz", @@ -167,14 +147,6 @@ } } }, - "d": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.0.tgz", - "integrity": "sha1-dUu1v+VUUdpppYuU1F9MWwRi1Y8=", - "requires": { - "es5-ext": "0.10.42" - } - }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -207,55 +179,6 @@ "jsbn": "0.1.1" } }, - "es5-ext": { - "version": "0.10.42", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.42.tgz", - "integrity": "sha512-AJxO1rmPe1bDEfSR6TJ/FgMFYuTBhR5R57KW58iCkYACMyFbrkqVyzXSurYoScDGvgyMpk7uRF/lPUPPTmsRSA==", - "requires": { - "es6-iterator": "2.0.3", - "es6-symbol": "3.1.1", - "next-tick": "1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1.0.0", - "es5-ext": "0.10.42", - "es6-symbol": "3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz", - "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=", - "requires": { - "d": "1.0.0", - "es5-ext": "0.10.42" - } - }, - "es6-weak-map": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.2.tgz", - "integrity": "sha1-XjqzIlH/0VOKH45f+hNXdy+S2W8=", - "requires": { - "d": "1.0.0", - "es5-ext": "0.10.42", - "es6-iterator": "2.0.3", - "es6-symbol": "3.1.1" - } - }, - "event-emitter": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", - "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=", - "requires": { - "d": "1.0.0", - "es5-ext": "0.10.42" - } - }, "execa": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", @@ -401,11 +324,6 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, - "is-promise": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" - }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -502,14 +420,6 @@ "yallist": "2.1.2" } }, - "lru-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", - "integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=", - "requires": { - "es5-ext": "0.10.42" - } - }, "mem": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", @@ -518,21 +428,6 @@ "mimic-fn": "1.2.0" } }, - "memoizee": { - "version": "0.4.12", - "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.12.tgz", - "integrity": "sha512-sprBu6nwxBWBvBOh5v2jcsGqiGLlL2xr2dLub3vR8dnE8YB17omwtm/0NSHl8jjNbcsJd5GMWJAnTSVe/O0Wfg==", - "requires": { - "d": "1.0.0", - "es5-ext": "0.10.42", - "es6-weak-map": "2.0.2", - "event-emitter": "0.3.5", - "is-promise": "2.1.0", - "lru-queue": "0.1.0", - "next-tick": "1.0.0", - "timers-ext": "0.1.5" - } - }, "mime-db": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", @@ -551,30 +446,11 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, - "moment": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.0.tgz", - "integrity": "sha512-1muXCh8jb1N/gHRbn9VDUBr0GYb8A/aVcHlII9QSB68a50spqEVLIGN6KVmCOnSvJrUhC0edGgKU5ofnGXdYdg==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, "node-fetch": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" }, - "note-log": { - "version": "2.1.11", - "resolved": "https://registry.npmjs.org/note-log/-/note-log-2.1.11.tgz", - "integrity": "sha1-DvEbJ2llJ2EfG5NHjaXCjyse7SQ=", - "requires": { - "cli-color": "1.2.0", - "moment": "2.22.0" - } - }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -816,15 +692,6 @@ "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" }, - "timers-ext": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.5.tgz", - "integrity": "sha512-tsEStd7kmACHENhsUPaxb8Jf8/+GZZxyNFQbZD07HQOyooOa6At1rQqjffgvg7n+dxscQa9cjjMdWhJtsP2sxg==", - "requires": { - "es5-ext": "0.10.42", - "next-tick": "1.0.0" - } - }, "tough-cookie": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", diff --git a/package.json b/package.json index 1988382..cdc5475 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "date-fns": "^1.29.0", "fs-extra": "^5.0.0", "node-fetch": "^2.1.2", - "note-log": "^2.1.11", "snoowrap": "^1.15.2", "url-pattern": "^1.0.3", "yargs": "^11.0.0"