DebaucheryLibrarian/ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for RedGifs and Reddit albums. Improved command line logger. Added rate limiters for reddit and host URLs.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-09-11 05:16:58 +02:00
parent bb06fe9763
commit de50d609f3
33 changed files with 586 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/sources/getIndex.js
View File

@@ -4,7 +4,8 @@ const config = require('config');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
const logger = require('../logger')(__filename);
const interpolate = require('../interpolate');
async function getIndex(user) {
const indexFilePath = interpolate(config.library.index.file, null, null, null, null, user, false);
@@ -14,7 +15,7 @@ async function getIndex(user) {
return yaml.safeLoad(indexFile);
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `No index file found for '${user.name}' at '${indexFilePath}'`);
logger.info(`No index file found for '${user.name}' at '${indexFilePath}'`);
return { profile: { image: null, description: null }, posts: [] };
}

23
src/sources/getPosts.js
View File

@@ -5,33 +5,38 @@ const Promise = require('bluebird');
const getIndex = require('./getIndex.js');
const curateUser = require('../curate/user.js');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').reddit;
async function getUser(username, reddit) {
try {
const user = await reddit.getUser(username).fetch();
const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
return curateUser(user);
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return {
name: username,
fallback: true,
};
}
};
}
const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
const post = await reddit
.getSubmission(postId)
.fetch();
const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch());
post.direct = true;
if (accUserPosts[post.author.name]) {
accUserPosts[post.author.name].posts = accUserPosts[post.author.name].posts.concat(post);
return accUserPosts;
return {
...accUserPosts,
[post.author.name]: {
...accUserPosts[post.author.name],
posts: [...accUserPosts[post.author.name].posts, post],
},
};
}
// don't attempt to fetch deleted user

25
src/sources/getUserPosts.js
View File

@@ -6,15 +6,18 @@ const getIndex = require('./getIndex.js');
const getArchivePostIds = require('../archives/getArchivePostIds.js');
const curateUser = require('../curate/user.js');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').reddit;
async function getUser(username, reddit) {
try {
const user = await reddit.getUser(username).fetch();
const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
console.log(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
return curateUser(user);
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return {
name: username,
@@ -25,18 +28,18 @@ async function getUser(username, reddit) {
async function getPosts(username, reddit, args) {
try {
const submissions = await reddit
const submissions = await limiter.schedule(async () => reddit
.getUser(username)
.getSubmissions({
sort: args.sort,
limit: Infinity,
});
}));
console.log(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
return submissions;
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
logger.warn(`Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return [];
}
@@ -45,7 +48,7 @@ async function getPosts(username, reddit, args) {
async function getArchivedPosts(username, posts, reddit) {
const postIds = await getArchivePostIds(username, posts.map(post => post.id));
return Promise.all(postIds.map(postId => reddit.getSubmission(postId).fetch()));
return Promise.all(postIds.map(postId => limiter.schedule(async () => reddit.getSubmission(postId).fetch())));
}
function getUserPostsWrap(reddit, args) {
@@ -78,12 +81,12 @@ function getUserPostsWrap(reddit, args) {
return null;
} catch (error) {
console.log(username, error);
logger.error(`Failed to fetch posts from 'username': ${error.message}`);
return null;
}
}, {
concurrency: 5,
concurrency: 10,
});
return users.reduce(
@@ -97,7 +100,7 @@ function getUserPostsWrap(reddit, args) {
{},
);
} catch (error) {
console.log(error);
logger.error(error);
throw error;
}