From fcb85f57c8b72b203ad77704a96dd9e65f09fa0d Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:56 +0200 Subject: [PATCH] Added watch-mode. Waiting for profile detail write to finalize before new watch cycle and capture details in index file. --- config/default.js | 4 ++ src/app.js | 14 +++++++ src/cli.js | 4 ++ src/fetch/content.js | 5 +-- src/save/profileDetails.js | 72 +++++++++++++++++++++++----------- src/save/save.js | 2 +- src/save/writeToIndex.js | 25 +++++++++++- src/sources/getIndexedPosts.js | 2 +- src/sources/getUserPosts.js | 2 +- 9 files changed, 99 insertions(+), 31 deletions(-) diff --git a/config/default.js b/config/default.js index 917b588..280888f 100644 --- a/config/default.js +++ b/config/default.js @@ -45,6 +45,10 @@ module.exports = { limit: 1000, avoidDuplicates: true, retries: 3, + watch: { + interval: 30, + ignoreErrors: true, + }, archives: { search: false, preview: true, diff --git a/src/app.js b/src/app.js index 9b02127..b19b545 100644 --- a/src/app.js +++ b/src/app.js @@ -39,6 +39,12 @@ function fetchSavePosts(userPosts, ep) { } async function initApp() { + function watch() { + console.log(`Watch-mode enabled, checking for new posts ${config.fetch.watch.interval} minutes from now.`); + + setTimeout(initApp, Math.ceil(config.fetch.watch.interval) * 1000 * 60); + } + const usersProvided = args.users && args.users.length; const postIdsProvided = args.posts && args.posts.length; @@ -53,8 +59,16 @@ async function initApp() { await ep.open(); await fetchSavePosts(userPosts, ep); await ep.close(); + + if (args.watch) { + watch(); + } } catch (error) { console.error(error); + + if (args.watch && config.fetch.watch.ignoreErrors) { + watch(); + } } return true; diff --git a/src/cli.js b/src/cli.js index c0c9f4e..61d9ca6 100644 --- a/src/cli.js +++ b/src/cli.js @@ -57,6 +57,10 @@ function getArgs() { describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed', type: 'boolean', }) + .option('watch', { + describe: 'Keep the process running and periodically check for new posts', + type: 'boolean', + }) .option('archives', { describe: 'Search archives for deleted posts', type: 'boolean', diff --git a/src/fetch/content.js b/src/fetch/content.js index ef3e1f3..cab6d6a 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -48,8 +48,7 @@ function getFilepath(item, post, user) { } async function fetchSaveContent(user, ep) { - // async, nothing depends on its success so don't await - saveProfileDetails(user); + const profilePaths = await saveProfileDetails(user); const posts = await Promise.map(user.posts, async (post) => { await Promise.reduce(post.content.items, async (accItems, originalItem, index) => { @@ -76,7 +75,7 @@ async function fetchSaveContent(user, ep) { return post; }); - return writeToIndex(posts, user); + return writeToIndex(posts, profilePaths, user); } module.exports = fetchSaveContent; diff --git a/src/save/profileDetails.js b/src/save/profileDetails.js index 9c46af1..06150e4 100644 --- a/src/save/profileDetails.js +++ b/src/save/profileDetails.js @@ -1,45 +1,71 @@ 'use strict'; const config = require('config'); -const urlPattern = require('url-pattern'); +const Promise = require('bluebird'); +const UrlPattern = require('url-pattern'); const interpolate = require('../interpolate.js'); const fetchItem = require('../fetch/item.js'); const textToStream = require('./textToStream.js'); const save = require('./save.js'); -function saveProfileDetails(user) { - if(config.library.profile.image && !user.fallback && !user.deleted) { +async function saveProfileImage(user) { + if (config.library.profile.image && !user.fallback && !user.deleted) { const image = user.profile ? user.profile.image : user.image; - if(config.library.profile.avoidAvatar && new urlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) { + if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) { console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}'`); - } else { - const filepath = interpolate(config.library.profile.image, user, null, { - // pass profile image as item to interpolate extension variable - url: image - }); - fetchItem(image, 0, {permalink: `https://reddit.com/user/${user.name}`}).then(stream => save(filepath, stream)).catch(error => { - console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`); - }); + return null; + } + + const filepath = interpolate(config.library.profile.image, user, null, { + // pass profile image as item to interpolate extension variable + url: image, + }); + + try { + const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` }) + const targets = await save(filepath, stream); + + return targets[0]; + } catch (error) { + console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`); + + return null; } } +} - if(config.library.profile.description && !user.fallback && !user.deleted) { - if(user.profile && user.profile.description) { +async function saveProfileDescription(user) { + if (config.library.profile.description && !user.fallback && !user.deleted) { + if (user.profile && user.profile.description) { const filepath = interpolate(config.library.profile.description, user); const stream = textToStream(user.profile.description); - save(filepath, stream).catch(error => { - console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`); - }); - } else { - console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`); - } - } + console.log(filepath); - return user; -}; + try { + const targets = await save(filepath, stream); + + return targets[0]; + } catch (error) { + console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`); + + return null; + } + } + + console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`); + + return null; + } +} + +async function saveProfileDetails(user) { + const [image, description] = await Promise.all([saveProfileImage(user), saveProfileDescription(user)]); + + return { image, description }; +} module.exports = saveProfileDetails; diff --git a/src/save/save.js b/src/save/save.js index 0cc525f..ee3c270 100644 --- a/src/save/save.js +++ b/src/save/save.js @@ -32,7 +32,7 @@ function pipeStreamToFile(target, stream, item) { stream.pipe(file); stream.on('error', reject); - stream.on('finish', () => { + stream.on('end', () => { if (item && item.mux) { console.log(`Temporarily saved '${target}', queued for muxing`); } else { diff --git a/src/save/writeToIndex.js b/src/save/writeToIndex.js index cbf8c8b..ce2f5a7 100644 --- a/src/save/writeToIndex.js +++ b/src/save/writeToIndex.js @@ -6,10 +6,11 @@ const yaml = require('js-yaml'); const interpolate = require('../interpolate.js'); -async function writeToIndex(posts, user) { +async function writeToIndex(posts, profilePaths, user) { const filename = interpolate(config.library.index.file, user, null, false); const now = new Date(); + /* // Individual posts are wrapped in [] to get a YAML array value for each individual item, allowing them to be joined manually with a newline // between each entry to improve human readability of the index while maintaining a valid YAML list const originalEntries = user.indexed.original.map(entry => yaml.safeDump([entry])); @@ -25,8 +26,28 @@ async function writeToIndex(posts, user) { }])); const entries = newAndUpdatedEntries.concat(originalEntries).join('\n'); + */ - return fs.writeFile(filename, entries); + const newAndUpdatedEntries = posts.concat(user.indexed.updated).map(post => ({ + id: post.id, + subreddit: post.subreddit, + permalink: post.permalink, + url: post.url, + hostId: post.host.id, + date: post.datetime, + indexed: now, + title: post.title, + })); + + const data = { + profile: { + image: profilePaths.image, + description: profilePaths.description, + }, + posts: newAndUpdatedEntries.concat(user.indexed.original), + }; + + return fs.writeFile(filename, yaml.safeDump(data)); } module.exports = writeToIndex; diff --git a/src/sources/getIndexedPosts.js b/src/sources/getIndexedPosts.js index 150c32f..5ebbbfa 100644 --- a/src/sources/getIndexedPosts.js +++ b/src/sources/getIndexedPosts.js @@ -16,7 +16,7 @@ async function getIndexedPosts(user) { } catch (error) { console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`); - return []; + return { profile: { image: null, description: null }, posts: [] }; } } diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index e469e37..a3bafcf 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -54,7 +54,7 @@ function getUserPostsWrap(reddit, args) { getPosts(username, reddit, args), ]); - const indexed = await getIndexedPosts(user); + const { posts: indexed } = await getIndexedPosts(user); if (args.archives) { posts.push(...await getArchivedPosts(username, posts, reddit));