Storing pHash. Fixed RedGIFs user agent.

src/save/writeToIndex.js

@@ -8,6 +8,18 @@ const interpolate = require('../interpolate');
 const save = require('./save');
 const logger = require('../logger')(__filename);
 
+function curatePosts(newPosts, indexedPosts) {
+    // console.log(indexedPosts, 'new');
+
+    const newIds = new Set(newPosts.map((post) => post.id));
+    const uniqueIndexedPosts = indexedPosts.filter((post) => !newIds.has(post.id));
+    const combinedPosts = newPosts.concat(uniqueIndexedPosts);
+
+    // console.log(combinedPosts);
+
+    return combinedPosts;
+}
+
 async function writeToIndex(posts, profilePaths, user, args) {
     const filepath = interpolate(config.library.index.file, null, null, null, null, user, false);
     const now = new Date();
@@ -24,6 +36,8 @@ async function writeToIndex(posts, profilePaths, user, args) {
             score: post.score,
             title: post.title,
             hash: post.hash,
+            phash: post.phash,
+            content: post.content,
         };
 
         if (post.previewFallback) {
@@ -34,11 +48,12 @@ async function writeToIndex(posts, profilePaths, user, args) {
     });
 
     const data = {
+        updated: new Date(),
         profile: {
             image: profilePaths.image,
             description: profilePaths.description,
         },
-        posts: newAndUpdatedEntries.concat(user.indexed.original),
+        posts: curatePosts(newAndUpdatedEntries, user.indexed.original),
     };
 
     if (!data.profile.image && !data.profile.description && !data.posts.length) {
@@ -46,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) {
     }
 
     try {
-        const yamlIndex = yaml.safeDump(data);
+        const yamlIndex = yaml.safeDump(data, { skipInvalid: true });
         const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8'));
 
         logger.info(`Saved index with ${posts.length} new posts for ${user.name}`);