'use strict'; const config = require('config'); const dissectLink = require('../dissectLink.js'); function curatePosts(posts, ignore) { const processed = new Set(); return posts.reduce((acc, post, index) => { const host = dissectLink(post.url); const ignoring = ignore ? ignore.find(prop => { return post[prop]; }) : null; if(ignoring) { console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' - ${post.url}`); return acc; } if(host) { if(config.fetch.avoidDuplicates && processed.has(host.id)) { console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${post.title}' - ${post.url}`); return acc; } processed.add(host.id); } return acc.concat({ id: post.id, index: index, title: post.title, text: post.selftext, user: post.user, permalink: 'https://reddit.com' + post.permalink, url: post.url, datetime: new Date(post.created_utc * 1000), subreddit: post.subreddit.display_name, preview: post.preview ? post.preview.images.map(image => image.source) : null, host }); }, []); }; module.exports = curatePosts;