DebaucheryLibrarian/ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored post retrieval so limit is applied per-user and ignores directly requested posts, and to start utilizing async/await.

Browse Source
This commit is contained in:
ThePendulum
2018-05-28 01:42:46 +02:00
parent 010470ca8c
commit e056acfbd3
8 changed files with 124 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/curate/posts.js
View File

@@ -2,54 +2,60 @@
const config = require('config');
const dissectLink = require('../dissectLink.js');
const omit = require('object.omit');
function curatePosts(posts, args) {
const curatePosts = (userPosts, args) => {
const processed = new Set();
return posts.reduce((acc, post, index) => {
return Object.values(userPosts).reduce((accPosts, user) => accPosts.concat(user.posts.reduce((accUserPosts, post, index) => {
// cut-off at limit, but don't count posts requested directly by ID
if(accUserPosts.length >= args.limit && !post.direct) {
return accUserPosts;
}
const host = dissectLink(post.url);
post.permalink = 'https://reddit.com' + post.permalink;
const permalink = 'https://reddit.com' + post.permalink;
const ignoring = args.ignore ? args.ignore.find(prop => {
return post[prop];
}) : null;
if(ignoring) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' (${post.permalink})`);
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' (${permalink})`);
return acc;
return accUserPosts;
}
if(host) {
if(config.fetch.avoidDuplicates && processed.has(host.id)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${post.url}' (${post.permalink})`);
console.log('\x1b[33m%s\x1b[0m', `Ignoring duplicate content '${post.url}' (cross-post, repost, or --post ID was already included) (${permalink})`);
return acc;
return accUserPosts;
}
if((args.include && !args.include.includes(host.label)) || (args.exclude && args.exclude.includes(host.label))) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring source '${host.label}' from post '${post.url}' (${post.permalink})`);
console.log('\x1b[33m%s\x1b[0m', `Ignoring source '${host.label}' from post '${post.url}' (${permalink})`);
return acc;
return accUserPosts;
}
processed.add(host.id);
}
return acc.concat({
return accUserPosts.concat({
id: post.id,
index: index,
title: post.title,
text: post.selftext,
user: post.user,
permalink: post.permalink,
user: omit(user, ['posts']),
permalink,
url: post.url,
datetime: new Date(post.created_utc * 1000),
subreddit: post.subreddit.display_name,
preview: post.preview ? post.preview.images.map(image => image.source) : null,
host
});
}, []);
}, [])), []);
};
module.exports = curatePosts;