From e087e470695e542f6e78d7e43f5bcda3bdbce953 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:56 +0200 Subject: [PATCH] No longer redownload profile image and description when indexed, unless --redownload-profile is specified. --- src/app.js | 2 +- src/cli.js | 4 +++ src/curate/posts.js | 1 + src/fetch/content.js | 4 +-- src/save/profileDetails.js | 32 ++++++++++++------- .../{getIndexedPosts.js => getIndex.js} | 4 +-- src/sources/getPosts.js | 6 ++-- src/sources/getUserPosts.js | 6 ++-- 8 files changed, 37 insertions(+), 22 deletions(-) rename src/sources/{getIndexedPosts.js => getIndex.js} (89%) diff --git a/src/app.js b/src/app.js index b19b545..6161d60 100644 --- a/src/app.js +++ b/src/app.js @@ -35,7 +35,7 @@ async function getCompleteUserPosts() { } function fetchSavePosts(userPosts, ep) { - return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep))); + return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep, args))); } async function initApp() { diff --git a/src/cli.js b/src/cli.js index 61d9ca6..770bbaa 100644 --- a/src/cli.js +++ b/src/cli.js @@ -57,6 +57,10 @@ function getArgs() { describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed', type: 'boolean', }) + .option('redownload-profile', { + describe: 'Ignore index file and force a redownload of the profile image and description', + type: 'boolean', + }) .option('watch', { describe: 'Keep the process running and periodically check for new posts', type: 'boolean', diff --git a/src/curate/posts.js b/src/curate/posts.js index 22b4d74..91fd153 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -149,6 +149,7 @@ const curatePosts = (userPosts, args) => Object.values(userPosts).reduce((accPos ...user, posts: curatedPosts.posts, indexed: { + profile: user.indexed.profile, original: indexedOriginal, updated: curatedPosts.indexedUpdated, oldest: indexed.oldest, diff --git a/src/fetch/content.js b/src/fetch/content.js index cab6d6a..2a710e0 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -47,8 +47,8 @@ function getFilepath(item, post, user) { : interpolate(config.library[type], user, post, item); } -async function fetchSaveContent(user, ep) { - const profilePaths = await saveProfileDetails(user); +async function fetchSaveContent(user, ep, args) { + const profilePaths = await saveProfileDetails(user, args); const posts = await Promise.map(user.posts, async (post) => { await Promise.reduce(post.content.items, async (accItems, originalItem, index) => { diff --git a/src/save/profileDetails.js b/src/save/profileDetails.js index 06150e4..72b3855 100644 --- a/src/save/profileDetails.js +++ b/src/save/profileDetails.js @@ -9,12 +9,18 @@ const fetchItem = require('../fetch/item.js'); const textToStream = require('./textToStream.js'); const save = require('./save.js'); -async function saveProfileImage(user) { +async function saveProfileImage(user, args) { + if (!args.redownloadProfile && user.indexed.profile.description) { + console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile image for '${user.name}' (https://reddit.com/user/${user.name})`); + + return user.indexed.profile.image; + } + if (config.library.profile.image && !user.fallback && !user.deleted) { const image = user.profile ? user.profile.image : user.image; if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) { - console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}'`); + console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}' (https://reddit.com/user/${user.name})`); return null; } @@ -25,45 +31,49 @@ async function saveProfileImage(user) { }); try { - const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` }) + const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` }); const targets = await save(filepath, stream); return targets[0]; } catch (error) { - console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`); + console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error} (https://reddit.com/user/${user.name})`); return null; } } } -async function saveProfileDescription(user) { +async function saveProfileDescription(user, args) { + if (!args.redownloadProfile && user.indexed.profile.description) { + console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile description for '${user.name}' (https://reddit.com/user/${user.name})`); + + return user.indexed.profile.description; + } + if (config.library.profile.description && !user.fallback && !user.deleted) { if (user.profile && user.profile.description) { const filepath = interpolate(config.library.profile.description, user); const stream = textToStream(user.profile.description); - console.log(filepath); - try { const targets = await save(filepath, stream); return targets[0]; } catch (error) { - console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`); + console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error} (https://reddit.com/user/${user.name})`); return null; } } - console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`); + console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}' (https://reddit.com/user/${user.name})`); return null; } } -async function saveProfileDetails(user) { - const [image, description] = await Promise.all([saveProfileImage(user), saveProfileDescription(user)]); +async function saveProfileDetails(user, args) { + const [image, description] = await Promise.all([saveProfileImage(user, args), saveProfileDescription(user, args)]); return { image, description }; } diff --git a/src/sources/getIndexedPosts.js b/src/sources/getIndex.js similarity index 89% rename from src/sources/getIndexedPosts.js rename to src/sources/getIndex.js index 5ebbbfa..193b875 100644 --- a/src/sources/getIndexedPosts.js +++ b/src/sources/getIndex.js @@ -6,7 +6,7 @@ const yaml = require('js-yaml'); const interpolate = require('../interpolate.js'); -async function getIndexedPosts(user) { +async function getIndex(user) { const indexFilePath = interpolate(config.library.index.file, user, null, null, false); try { @@ -20,4 +20,4 @@ async function getIndexedPosts(user) { } } -module.exports = getIndexedPosts; +module.exports = getIndex; diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index db201c3..724ef6c 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -2,7 +2,7 @@ const Promise = require('bluebird'); -const getIndexedPosts = require('./getIndexedPosts.js'); +const getIndex = require('./getIndex.js'); const curateUser = require('../curate/user.js'); const getUser = async (username, reddit) => { @@ -38,9 +38,9 @@ const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { } const user = await getUser(post.author.name, reddit); - const indexed = await getIndexedPosts(user); + const { profile, posts: indexed } = await getIndexedPosts(user); - return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { original: indexed, updated: [] } } }; + return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { profile, original: indexed, updated: [] } } }; }), userPosts); }; diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index a3bafcf..f6e40fa 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -5,7 +5,7 @@ const Promise = require('bluebird'); const fs = require('fs-extra'); const yaml = require('js-yaml'); -const getIndexedPosts = require('./getIndexedPosts.js'); +const getIndex = require('./getIndex.js'); const getArchivePostIds = require('../archives/getArchivePostIds.js'); const curateUser = require('../curate/user.js'); const interpolate = require('../interpolate.js'); @@ -54,14 +54,14 @@ function getUserPostsWrap(reddit, args) { getPosts(username, reddit, args), ]); - const { posts: indexed } = await getIndexedPosts(user); + const { profile, posts: indexed } = await getIndex(user); if (args.archives) { posts.push(...await getArchivedPosts(username, posts, reddit)); } if (posts.length) { - return { ...user, posts, indexed: { original: indexed, updated: [] } }; + return { ...user, posts, indexed: { profile, original: indexed, updated: [] } }; } return null;