Added support for fetching individual posts. Improved use of yargs, --help now available. Refactored main app flow.

This commit is contained in:
2018-04-29 02:02:34 +02:00
parent 45182c07db
commit 05b5bee7ea
7 changed files with 150 additions and 95 deletions

46
src/curate/posts.js Normal file
View File

@@ -0,0 +1,46 @@
'use strict';
const config = require('config');
const dissectLink = require('../dissectLink.js');
function curatePosts(posts, ignore) {
const processed = new Set();
return posts.reduce((acc, post, index) => {
const host = dissectLink(post.url);
const ignoring = ignore ? ignore.find(prop => {
return post[prop];
}) : null;
if(ignoring) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' - ${post.url}`);
return acc;
}
if(host) {
if(config.fetch.avoidDuplicates && processed.has(host.id)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${post.title}' - ${post.url}`);
return acc;
}
processed.add(host.id);
}
return acc.concat({
id: post.id,
index: index,
title: post.title,
text: post.selftext,
user: post.user,
permalink: 'https://reddit.com' + post.permalink,
url: post.url,
datetime: new Date(post.created_utc * 1000),
subreddit: post.subreddit.display_name,
host
});
}, []);
};
module.exports = curatePosts;

View File

@@ -1,48 +0,0 @@
'use strict';
const config = require('config');
const dissectLink = require('../dissectLink.js');
function curateSubmissions(submissions, ignore) {
const processed = new Set();
return submissions.reduce((acc, submission, index) => {
const host = dissectLink(submission.url);
const ignoring = ignore.find(prop => {
return submission[prop];
});
if(ignoring) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${submission.title}' - ${submission.url}`);
return acc;
}
if(host) {
if(config.fetch.avoidDuplicates && processed.has(host.id)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
return acc;
}
processed.add(host.id);
}
const curatedSubmission = {
id: submission.id,
index: index,
title: submission.title,
text: submission.selftext,
user: submission.author.name,
permalink: 'https://reddit.com' + submission.permalink,
url: submission.url,
datetime: new Date(submission.created_utc * 1000),
subreddit: submission.subreddit.display_name,
host
};
return acc.concat(curatedSubmission);
}, []);
};
module.exports = curateSubmissions;