ripunzel/src/curate/posts.js

'use strict';

const config = require('config');
const omit = require('object.omit');

const dissectLink = require('../dissectLink.js');
const hashPost = require('./hashPost.js');

const { isAfter, isBefore, isEqual } = require('date-fns');

function report(curatedPosts, indexed, user, args) {
    const { posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount } = curatedPosts;

    if (indexedUpdated.length > 0) {
        console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`);
    }

    if (tooOldCount > 0) {
        console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`);
    }

    if (tooRecentCount > 0) {
        console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooRecentCount} newer posts for '${user.name}' for specified date limit '${args.before}'`);
    }

    if (beforeIndexedCount > 0) {
        console.log(
            '\x1b[33m%s\x1b[0m',
            `Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`
        );
    }

    if (afterIndexedCount > 0) {
        console.log(
            '\x1b[33m%s\x1b[0m',
            `Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`
        );
    }
};

function curatePost(acc, post, user, index, indexed, processed, args) {
    const host = dissectLink(post.url);
    const permalink = `https://reddit.com${post.permalink}`;

    const curatedPost = {
        id: post.id,
        index,
        title: post.title,
        text: post.selftext,
        user: omit(user, ['posts']),
        permalink,
        url: post.url,
        datetime: new Date(post.created_utc * 1000),
        subreddit: post.subreddit.display_name,
        preview: post.preview ? post.preview.images.map(image => image.source) : null,
        host,
        hash: hashPost(post),
    };

    if (indexed.entries.length) {
        if (!args.redownload && indexed.entries.find(entry => entry.id === post.id)) {
            return { ...acc, indexedUpdated: [...acc.indexedUpdated, curatedPost] };
        }

        if (args.afterIndexed && (isBefore(curatedPost.datetime, indexed[args.afterIndexed].date) || isEqual(curatedPost.datetime, indexed[args.afterIndexed].date))) {
            return { ...acc, beforeIndexedCount: acc.beforeIndexedCount + 1 };
        }

        if (args.beforeIndexed && (isAfter(curatedPost.datetime, indexed[args.beforeIndexed].date) || isEqual(curatedPost.datetime, indexed[args.beforeIndexed].date))) {
            return { ...acc, afterIndexedCount: acc.afterIndexedCount + 1 };
        }
    }

    if (args.after && (isBefore(curatedPost.datetime, args.after) || isEqual(curatedPost.datetime, args.after))) {
        return { ...acc, tooOldCount: acc.tooOldCount + 1 };
    }

    if (args.before && (isAfter(curatedPost.datetime, args.before) || isEqual(curatedPost.datetime, args.before))) {
        return { ...acc, tooRecentCount: acc.tooRecentCount + 1 };
    }

    // cut-off at limit, but don't count posts requested directly by ID
    if (acc.posts.length >= args.limit && !post.direct) {
        return acc;
    }

    const ignoring = args.ignore ? args.ignore.find(prop => post[prop]) : null;

    if (ignoring) {
        console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' (${permalink})`);

        return acc;
    }

    if (host) {
        const hostIncludes = args.include && !args.include.includes(host.label);
        const hostExcluded = args.exclude && args.exclude.includes(host.label);

        if (hostIncludes || hostExcluded) {
            console.log(
                '\x1b[33m%s\x1b[0m',
                `Ignoring source '${host.label}' from post '${post.url}' (${permalink})`,
            );

            return acc;
        }

        if (config.fetch.avoidDuplicates && processed.has(host.id)) {
            console.log(
                '\x1b[33m%s\x1b[0m',
                `Ignoring duplicate content '${post.url}' (cross-post, repost, or superfluous --post ID) (${permalink})`,
            );

            return acc;
        }

        processed.add(host.id);
    }

    return { ...acc, posts: [...acc.posts, curatedPost] };
}

const curatePosts = (userPosts, args) => Object.values(userPosts).reduce((accPosts, user) => {
    const processed = new Set();
    const indexedByDate = user.indexed.original.sort((entryA, entryB) => new Date(entryA.date) - new Date(entryB.date));

    const indexed = {
        entries: indexedByDate,
        oldest: indexedByDate.slice(0, 1)[0],
        latest: indexedByDate.slice(-1)[0],
    };

    const curatedPosts = user.posts.reduce((accUserPosts, post, index) => curatePost(accUserPosts, post, user, index, indexed, processed, args), {
        posts: [],
        indexedUpdated: [],
        tooOldCount: 0,
        tooRecentCount: 0,
        beforeIndexedCount: 0,
        afterIndexedCount: 0,
    });

    report(curatedPosts, indexed, user, args);

    const indexedOriginal = user.indexed.original.filter(entry => !curatedPosts.indexedUpdated.find(post => post.id === entry.id));

    return {
        ...accPosts,
        [user.name]: {
            ...user,
            posts: curatedPosts.posts,
            indexed: {
                original: indexedOriginal,
                updated: curatedPosts.indexedUpdated,
                oldest: indexed.oldest,
                latest: indexed.latest,
            },
        },
    };
}, {});

module.exports = curatePosts;