'use strict'; const config = require('config'); const dissectLink = require('../dissectLink.js'); function curateSubmissions(submissions) { const processed = new Set(); return submissions.reduce((acc, submission, index) => { const host = dissectLink(submission.url); if(host) { if(config.fetch.avoidDuplicates && processed.has(host.id)) { console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`); return acc; } processed.add(host.id); } const curatedSubmission = { id: submission.id, index: index, title: submission.title, text: submission.selftext, user: submission.author.name, permalink: 'https://reddit.com' + submission.permalink, url: submission.url, datetime: new Date(submission.created_utc * 1000), subreddit: submission.subreddit.display_name, host }; return acc.concat(curatedSubmission); }, []); }; module.exports = curateSubmissions;