48 lines
1.3 KiB
JavaScript
48 lines
1.3 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const dissectLink = require('../dissectLink.js');
|
|
|
|
function curatePosts(posts, ignore) {
|
|
const processed = new Set();
|
|
|
|
return posts.reduce((acc, post, index) => {
|
|
const host = dissectLink(post.url);
|
|
const ignoring = ignore ? ignore.find(prop => {
|
|
return post[prop];
|
|
}) : null;
|
|
|
|
if(ignoring) {
|
|
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' - ${post.url}`);
|
|
|
|
return acc;
|
|
}
|
|
|
|
if(host) {
|
|
if(config.fetch.avoidDuplicates && processed.has(host.id)) {
|
|
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${post.title}' - ${post.url}`);
|
|
|
|
return acc;
|
|
}
|
|
|
|
processed.add(host.id);
|
|
}
|
|
|
|
return acc.concat({
|
|
id: post.id,
|
|
index: index,
|
|
title: post.title,
|
|
text: post.selftext,
|
|
user: post.user,
|
|
permalink: 'https://reddit.com' + post.permalink,
|
|
url: post.url,
|
|
datetime: new Date(post.created_utc * 1000),
|
|
subreddit: post.subreddit.display_name,
|
|
preview: post.preview ? post.preview.images.map(image => image.source) : null,
|
|
host
|
|
});
|
|
}, []);
|
|
};
|
|
|
|
module.exports = curatePosts;
|