From 86002ef00b70aef79fc0e431e29ed2dc2903d512 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:56 +0200 Subject: [PATCH] Accounting for indexed posts. --- src/curate/posts.js | 18 +++++++++--------- src/interpolate.js | 1 - src/save/writeToIndex.js | 6 +++--- src/sources/getUserPosts.js | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/curate/posts.js b/src/curate/posts.js index 65678a1..149395a 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -25,8 +25,8 @@ function curatePost(acc, post, user, index, processed, args) { hash: hashPost(post), }; - if (user.indexed.find(entry => entry.id === post.id)) { - return { ...acc, indexed: { ...acc.indexed, [post.id]: curatedPost } }; + if (user.indexed.original.find(entry => entry.id === post.id)) { + return { ...acc, indexedUpdated: [...acc.indexedUpdated, curatedPost] }; } // cut-off at limit, but don't count posts requested directly by ID @@ -73,16 +73,16 @@ function curatePost(acc, post, user, index, processed, args) { const curatePosts = (userPosts, args) => Object.values(userPosts).reduce((accPosts, user) => { const processed = new Set(); - const { posts, indexed } = user.posts.reduce((accUserPosts, post, index) => - curatePost(accUserPosts, post, user, index, processed, args), { posts: [], indexed: {} }); + const { posts, indexedUpdated } = user.posts.reduce((accUserPosts, post, index) => + curatePost(accUserPosts, post, user, index, processed, args), { posts: [], indexedUpdated: [] }); - const indexedLength = Object.keys(indexed).length; - - if (indexedLength > 0) { - console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedLength} indexed posts for '${user.name}'`); + if (indexedUpdated.length > 0) { + console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`); } - return { ...accPosts, [user.name]: { ...user, posts } }; + const indexedOriginal = user.indexed.original.filter(entry => !indexedUpdated.find(post => post.id === entry.id)); + + return { ...accPosts, [user.name]: { ...user, posts, indexed: { original: indexedOriginal, updated: indexedUpdated } } }; }, {}); module.exports = curatePosts; diff --git a/src/interpolate.js b/src/interpolate.js index ad03496..6eb7aeb 100644 --- a/src/interpolate.js +++ b/src/interpolate.js @@ -5,7 +5,6 @@ const path = require('path'); const url = require('url'); const dateFns = require('date-fns'); const mime = require('mime-types'); -const dotty = require('dotty'); function interpolate(pattern, user, post, item, strip = true, dateFormat = config.library.dateFormat) { const vars = { diff --git a/src/save/writeToIndex.js b/src/save/writeToIndex.js index b45f8e4..cbf8c8b 100644 --- a/src/save/writeToIndex.js +++ b/src/save/writeToIndex.js @@ -12,8 +12,8 @@ async function writeToIndex(posts, user) { // Individual posts are wrapped in [] to get a YAML array value for each individual item, allowing them to be joined manually with a newline // between each entry to improve human readability of the index while maintaining a valid YAML list - const oldEntries = user.indexed.map(entry => yaml.safeDump([entry])); - const newEntries = posts.map(post => yaml.safeDump([{ + const originalEntries = user.indexed.original.map(entry => yaml.safeDump([entry])); + const newAndUpdatedEntries = posts.concat(user.indexed.updated).map(post => yaml.safeDump([{ id: post.id, subreddit: post.subreddit, permalink: post.permalink, @@ -24,7 +24,7 @@ async function writeToIndex(posts, user) { title: post.title, }])); - const entries = newEntries.concat(oldEntries).join('\n'); + const entries = newAndUpdatedEntries.concat(originalEntries).join('\n'); return fs.writeFile(filename, entries); } diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 607be75..1fa53fa 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -74,7 +74,7 @@ function getUserPostsWrap(reddit, args) { } if (posts.length) { - return { ...user, posts, indexed }; + return { ...user, posts, indexed: { original: indexed, updated: [] } }; } return null;