Added file save patterns, fetch retry and gfycat support.

This commit is contained in:
DebaucheryLibrarian 2024-09-11 05:16:53 +02:00
parent 8287286a36
commit 31ef234a71
9 changed files with 127 additions and 24 deletions

13
app.js
View File

@ -9,19 +9,20 @@ const methods = require('./methods/methods.js');
const dissectLink = require('./dissectLink.js');
const fetchContent = require('./fetchContent.js');
const reddit = new snoowrap(config.api);
const user = yargs.user;
const reddit = new snoowrap(config.reddit);
reddit.getUser(user).getSubmissions({
sort: 'top'
reddit.getUser(yargs.user).getSubmissions({
sort: 'top',
limit: Infinity
}).then(submissions => {
const curatedPosts = submissions.map(submission => {
return {
id: submission.id,
title: submission.title,
user: submission.author.name,
permalink: submission.permalink,
url: submission.url,
datetime: submission.created_utc,
datetime: submission.created_utc * 1000,
subreddit: submission.subreddit.display_name,
host: dissectLink(submission.url)
};
@ -40,6 +41,6 @@ reddit.getUser(user).getSubmissions({
return acc;
}, []));
}).then(posts => fetchContent(posts, user)).catch(error => {
}).then(posts => fetchContent(posts)).catch(error => {
note(error);
});

View File

@ -8,6 +8,9 @@ const hosts = [{
}, {
method: 'imgurAlbum',
pattern: new urlPattern('http(s)\\://imgur.com/:type/:id')
}, {
method: 'gfycat',
pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)')
}];
module.exports = function dissectLink(url) {

View File

@ -2,45 +2,105 @@
const fs = require('fs-extra');
const path = require('path');
const config = require('config');
const fetch = require('node-fetch');
const dateFns = require('date-fns');
function saveToDisk(buffer, item, index, filename, post, user) {
const extensions = {
'image/jpeg': '.jpg',
'image/gif': '.gif',
'video/mp4': '.mp4',
'video/webm': '.webm'
};
function interpolate(path, post, item, index) {
const dateFormat = config.patterns.date || 'YYYYMMDD';
const vars = {
$postId: post.id,
$postTitle: post.title,
$postUser: post.user,
$postDate: dateFns.format(post.datetime, dateFormat)
};
if(post.content.album) {
Object.assign(vars, {
$albumId: post.content.album.id,
$albumTitle: post.content.album.title,
$albumDescription: post.content.album.description,
$albumDate: dateFns.format(post.content.album.datetime, dateFormat)
});
}
if(item) {
Object.assign(vars, {
$itemId: item.id,
$itemTitle: item.title,
$itemDescription: item.description,
$itemDate: dateFns.format(item.datetime, dateFormat),
$itemIndex: index + config.patterns.indexOffset,
$ext: extensions[item.type]
});
}
return Object.entries(vars).reduce((acc, [key, value], index) => {
// strip slashes for filesystem compatability
value = (value || '').toString().replace(/\//g, config.patterns.slashSubstitute);
return acc.replace(key, value);
}, path);
};
function saveItemToDisk(buffer, item, index, filename, post) {
const stream = fs.createWriteStream(filename);
return new Promise((resolve, reject) => {
buffer.body.pipe(stream).on('error', error => {
reject(error);
}).on('finish', () => {
console.log(`Fetched and saved ${filename}`);
console.log(`Saved '${filename}'`);
resolve(filename);
});
});
};
function fetchItem(item, index, post, user) {
function fetchItem(item, index, post, attempt) {
function retry(error) {
console.log(error);
if(attempt < 3) {
console.log('Retrying...');
return fetchItem(item, index, post, ++attempt);
}
};
return fetch(item.url).then(res => {
return res.ok ? res : Promise.reject(`Failed to fetch ${item.url}`);
});
}).then(res => {
console.log(`Fetched '${item.url}'`);
return res;
}).catch(retry);
};
module.exports = function(posts, user) {
module.exports = function(posts) {
return Promise.all(posts.map(post => {
const filepath = post.content.album ? `output/${user}/${post.content.album.datetime} - ${post.content.album.id} - ${post.title}/` : `output/${user}/`;
return Promise.resolve().then(() => {
if(post.content.album) {
return fs.ensureDir(filepath);
};
}).then(() => {
return Promise.all(post.content.items.map((item, index) => {
return fetchItem(item, index, post, user).then(buffer => Object.assign(item, {buffer}));
return fetchItem(item, index, post, 0).then(buffer => Object.assign(item, {buffer}));
}));
}).then(items => {
return Promise.all(items.map((item, index) => {
const filename = post.content.album ? `${filepath}${index + 1} - ${item.id}.jpg` : `${filepath}${item.datetime} - ${item.id}.jpg`;
const type = item.type.split('/')[0];
const filepath = post.content.album ? interpolate(config.patterns.album[type], post, item, index) : interpolate(config.patterns[type], post, item, index);
return saveToDisk(item.buffer, item, index, filename, post, user)
return Promise.resolve().then(() => {
return fs.ensureDir(path.dirname(filepath));
}).then(() => {
return saveItemToDisk(item.buffer, item, index, filepath, post);
});
}));
});
}));

31
methods/gfycat.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
const note = require('note-log');
const util = require('util');
const config = require('config');
const fetch = require('node-fetch');
function imgurImage(post) {
return fetch(`https://api.gfycat.com/v1/gfycats/${post.host.id}`, {
headers: {
'Authorization': `Bearer ${config.methods.gfycat.key}`
}
}).then(res => res.json()).then(res => {
return {
album: null,
items: [{
id: res.gfyItem.gfyName,
url: res.gfyItem.webmUrl,
title: res.gfyItem.title,
description: res.gfyItem.description,
type: 'video/webm',
datetime: res.gfyItem.createDate * 1000,
original: res.gfyItem
}]
};
}).catch(error => {
note(error);
});
};
module.exports = imgurImage;

View File

@ -17,7 +17,7 @@ function imgurAlbum(post) {
url: res.data.link,
title: res.data.title,
description: res.data.description,
datetime: res.data.datetime,
datetime: res.data.datetime * 1000,
original: res.data
},
items: res.data.images.map(item => ({
@ -25,7 +25,7 @@ function imgurAlbum(post) {
url: item.link,
title: item.title,
description: item.description,
datetime: item.datetime,
datetime: item.datetime * 1000,
type: item.type,
original: item
}))

View File

@ -19,7 +19,7 @@ function imgurImage(post) {
title: res.data.title,
description: res.data.description,
type: res.data.type,
datetime: res.data.datetime,
datetime: res.data.datetime * 1000,
original: res.data
}]
};

View File

@ -2,8 +2,10 @@
const imgurImage = require('./imgurImage.js');
const imgurAlbum = require('./imgurAlbum.js');
const gfycat= require('./gfycat.js');
module.exports = {
imgurImage: imgurImage,
imgurAlbum: imgurAlbum
imgurAlbum: imgurAlbum,
gfycat: gfycat
};

5
package-lock.json generated
View File

@ -183,6 +183,11 @@
"assert-plus": "1.0.0"
}
},
"date-fns": {
"version": "1.29.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
"integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
},
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",

View File

@ -23,6 +23,7 @@
"license": "ISC",
"dependencies": {
"config": "^1.30.0",
"date-fns": "^1.29.0",
"fs-extra": "^5.0.0",
"node-fetch": "^2.1.2",
"note-log": "^2.1.11",