From 31ef234a7166c52534f920cf1e0727beaa7e1d95 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:53 +0200 Subject: [PATCH] Added file save patterns, fetch retry and gfycat support. --- app.js | 13 ++++--- dissectLink.js | 3 ++ fetchContent.js | 88 ++++++++++++++++++++++++++++++++++++------- methods/gfycat.js | 31 +++++++++++++++ methods/imgurAlbum.js | 4 +- methods/imgurImage.js | 2 +- methods/methods.js | 4 +- package-lock.json | 5 +++ package.json | 1 + 9 files changed, 127 insertions(+), 24 deletions(-) create mode 100644 methods/gfycat.js diff --git a/app.js b/app.js index 03b26df..663f0b4 100644 --- a/app.js +++ b/app.js @@ -9,19 +9,20 @@ const methods = require('./methods/methods.js'); const dissectLink = require('./dissectLink.js'); const fetchContent = require('./fetchContent.js'); -const reddit = new snoowrap(config.api); -const user = yargs.user; +const reddit = new snoowrap(config.reddit); -reddit.getUser(user).getSubmissions({ - sort: 'top' +reddit.getUser(yargs.user).getSubmissions({ + sort: 'top', + limit: Infinity }).then(submissions => { const curatedPosts = submissions.map(submission => { return { id: submission.id, title: submission.title, + user: submission.author.name, permalink: submission.permalink, url: submission.url, - datetime: submission.created_utc, + datetime: submission.created_utc * 1000, subreddit: submission.subreddit.display_name, host: dissectLink(submission.url) }; @@ -40,6 +41,6 @@ reddit.getUser(user).getSubmissions({ return acc; }, [])); -}).then(posts => fetchContent(posts, user)).catch(error => { +}).then(posts => fetchContent(posts)).catch(error => { note(error); }); diff --git a/dissectLink.js b/dissectLink.js index 0564b77..33603de 100644 --- a/dissectLink.js +++ b/dissectLink.js @@ -8,6 +8,9 @@ const hosts = [{ }, { method: 'imgurAlbum', pattern: new urlPattern('http(s)\\://imgur.com/:type/:id') +}, { + method: 'gfycat', + pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)') }]; module.exports = function dissectLink(url) { diff --git a/fetchContent.js b/fetchContent.js index 0f71220..c014a34 100644 --- a/fetchContent.js +++ b/fetchContent.js @@ -2,45 +2,105 @@ const fs = require('fs-extra'); const path = require('path'); +const config = require('config'); const fetch = require('node-fetch'); +const dateFns = require('date-fns'); -function saveToDisk(buffer, item, index, filename, post, user) { +const extensions = { + 'image/jpeg': '.jpg', + 'image/gif': '.gif', + 'video/mp4': '.mp4', + 'video/webm': '.webm' +}; + +function interpolate(path, post, item, index) { + const dateFormat = config.patterns.date || 'YYYYMMDD'; + + const vars = { + $postId: post.id, + $postTitle: post.title, + $postUser: post.user, + $postDate: dateFns.format(post.datetime, dateFormat) + }; + + if(post.content.album) { + Object.assign(vars, { + $albumId: post.content.album.id, + $albumTitle: post.content.album.title, + $albumDescription: post.content.album.description, + $albumDate: dateFns.format(post.content.album.datetime, dateFormat) + }); + } + + if(item) { + Object.assign(vars, { + $itemId: item.id, + $itemTitle: item.title, + $itemDescription: item.description, + $itemDate: dateFns.format(item.datetime, dateFormat), + $itemIndex: index + config.patterns.indexOffset, + $ext: extensions[item.type] + }); + } + + return Object.entries(vars).reduce((acc, [key, value], index) => { + // strip slashes for filesystem compatability + value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute); + + return acc.replace(key, value); + }, path); +}; + +function saveItemToDisk(buffer, item, index, filename, post) { const stream = fs.createWriteStream(filename); return new Promise((resolve, reject) => { buffer.body.pipe(stream).on('error', error => { reject(error); }).on('finish', () => { - console.log(`Fetched and saved ${filename}`); + console.log(`Saved '${filename}'`); resolve(filename); }); }); }; -function fetchItem(item, index, post, user) { +function fetchItem(item, index, post, attempt) { + function retry(error) { + console.log(error); + + if(attempt < 3) { + console.log('Retrying...'); + + return fetchItem(item, index, post, ++attempt); + } + }; + return fetch(item.url).then(res => { return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`); - }); + }).then(res => { + console.log(`Fetched '${item.url}'`); + + return res; + }).catch(retry); }; -module.exports = function(posts, user) { +module.exports = function(posts) { return Promise.all(posts.map(post => { - const filepath = post.content.album ? `output/${user}/${post.content.album.datetime} - ${post.content.album.id} - ${post.title}/` : `output/${user}/`; - return Promise.resolve().then(() => { - if(post.content.album) { - return fs.ensureDir(filepath); - }; - }).then(() => { return Promise.all(post.content.items.map((item, index) => { - return fetchItem(item, index, post, user).then(buffer => Object.assign(item, {buffer})); + return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer})); })); }).then(items => { return Promise.all(items.map((item, index) => { - const filename = post.content.album ? `${filepath}${index + 1} - ${item.id}.jpg` : `${filepath}${item.datetime} - ${item.id}.jpg`; + const type = item.type.split('/')[0]; + const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item, index) : interpolate(config.patterns[type], post, item, index); - return saveToDisk(item.buffer, item, index, filename, post, user) + return Promise.resolve().then(() => { + return fs.ensureDir(path.dirname(filepath)); + }).then(() => { + return saveItemToDisk(item.buffer, item, index, filepath, post); + }); })); }); })); diff --git a/methods/gfycat.js b/methods/gfycat.js new file mode 100644 index 0000000..3cd91df --- /dev/null +++ b/methods/gfycat.js @@ -0,0 +1,31 @@ + 'use strict'; + +const note = require('note-log'); +const util = require('util'); +const config = require('config'); +const fetch = require('node-fetch'); + +function imgurImage(post) { + return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, { + headers: { + 'Authorization': `Bearer ${config.methods.gfycat.key}` + } + }).then(res => res.json()).then(res => { + return { + album: null, + items: [{ + id: res.gfyItem.gfyName, + url: res.gfyItem.webmUrl, + title: res.gfyItem.title, + description: res.gfyItem.description, + type: 'video/webm', + datetime: res.gfyItem.createDate * 1000, + original: res.gfyItem + }] + }; + }).catch(error => { + note(error); + }); +}; + +module.exports = imgurImage; diff --git a/methods/imgurAlbum.js b/methods/imgurAlbum.js index c7c37e6..a3d8e5e 100644 --- a/methods/imgurAlbum.js +++ b/methods/imgurAlbum.js @@ -17,7 +17,7 @@ function imgurAlbum(post) { url: res.data.link, title: res.data.title, description: res.data.description, - datetime: res.data.datetime, + datetime: res.data.datetime * 1000, original: res.data }, items: res.data.images.map(item => ({ @@ -25,7 +25,7 @@ function imgurAlbum(post) { url: item.link, title: item.title, description: item.description, - datetime: item.datetime, + datetime: item.datetime * 1000, type: item.type, original: item })) diff --git a/methods/imgurImage.js b/methods/imgurImage.js index 3ebcd12..ebdd338 100644 --- a/methods/imgurImage.js +++ b/methods/imgurImage.js @@ -19,7 +19,7 @@ function imgurImage(post) { title: res.data.title, description: res.data.description, type: res.data.type, - datetime: res.data.datetime, + datetime: res.data.datetime * 1000, original: res.data }] }; diff --git a/methods/methods.js b/methods/methods.js index 3e9fd9b..317fe41 100644 --- a/methods/methods.js +++ b/methods/methods.js @@ -2,8 +2,10 @@ const imgurImage = require('./imgurImage.js'); const imgurAlbum = require('./imgurAlbum.js'); +const gfycat= require('./gfycat.js'); module.exports = { imgurImage: imgurImage, - imgurAlbum: imgurAlbum + imgurAlbum: imgurAlbum, + gfycat: gfycat }; diff --git a/package-lock.json b/package-lock.json index abf9e96..acfd0f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -183,6 +183,11 @@ "assert-plus": "1.0.0" } }, + "date-fns": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz", + "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==" + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", diff --git a/package.json b/package.json index 44b4dfb..4833d47 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "license": "ISC", "dependencies": { "config": "^1.30.0", + "date-fns": "^1.29.0", "fs-extra": "^5.0.0", "node-fetch": "^2.1.2", "note-log": "^2.1.11",