No longer redownload profile image and description when indexed, unless --redownload-profile is specified.
This commit is contained in:
parent
fcb85f57c8
commit
e087e47069
|
@ -35,7 +35,7 @@ async function getCompleteUserPosts() {
|
|||
}
|
||||
|
||||
function fetchSavePosts(userPosts, ep) {
|
||||
return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep)));
|
||||
return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep, args)));
|
||||
}
|
||||
|
||||
async function initApp() {
|
||||
|
|
|
@ -57,6 +57,10 @@ function getArgs() {
|
|||
describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
|
||||
type: 'boolean',
|
||||
})
|
||||
.option('redownload-profile', {
|
||||
describe: 'Ignore index file and force a redownload of the profile image and description',
|
||||
type: 'boolean',
|
||||
})
|
||||
.option('watch', {
|
||||
describe: 'Keep the process running and periodically check for new posts',
|
||||
type: 'boolean',
|
||||
|
|
|
@ -149,6 +149,7 @@ const curatePosts = (userPosts, args) => Object.values(userPosts).reduce((accPos
|
|||
...user,
|
||||
posts: curatedPosts.posts,
|
||||
indexed: {
|
||||
profile: user.indexed.profile,
|
||||
original: indexedOriginal,
|
||||
updated: curatedPosts.indexedUpdated,
|
||||
oldest: indexed.oldest,
|
||||
|
|
|
@ -47,8 +47,8 @@ function getFilepath(item, post, user) {
|
|||
: interpolate(config.library[type], user, post, item);
|
||||
}
|
||||
|
||||
async function fetchSaveContent(user, ep) {
|
||||
const profilePaths = await saveProfileDetails(user);
|
||||
async function fetchSaveContent(user, ep, args) {
|
||||
const profilePaths = await saveProfileDetails(user, args);
|
||||
|
||||
const posts = await Promise.map(user.posts, async (post) => {
|
||||
await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
|
||||
|
|
|
@ -9,12 +9,18 @@ const fetchItem = require('../fetch/item.js');
|
|||
const textToStream = require('./textToStream.js');
|
||||
const save = require('./save.js');
|
||||
|
||||
async function saveProfileImage(user) {
|
||||
async function saveProfileImage(user, args) {
|
||||
if (!args.redownloadProfile && user.indexed.profile.description) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return user.indexed.profile.image;
|
||||
}
|
||||
|
||||
if (config.library.profile.image && !user.fallback && !user.deleted) {
|
||||
const image = user.profile ? user.profile.image : user.image;
|
||||
|
||||
if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}'`);
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
@ -25,45 +31,49 @@ async function saveProfileImage(user) {
|
|||
});
|
||||
|
||||
try {
|
||||
const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` })
|
||||
const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` });
|
||||
const targets = await save(filepath, stream);
|
||||
|
||||
return targets[0];
|
||||
} catch (error) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function saveProfileDescription(user) {
|
||||
async function saveProfileDescription(user, args) {
|
||||
if (!args.redownloadProfile && user.indexed.profile.description) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return user.indexed.profile.description;
|
||||
}
|
||||
|
||||
if (config.library.profile.description && !user.fallback && !user.deleted) {
|
||||
if (user.profile && user.profile.description) {
|
||||
const filepath = interpolate(config.library.profile.description, user);
|
||||
const stream = textToStream(user.profile.description);
|
||||
|
||||
console.log(filepath);
|
||||
|
||||
try {
|
||||
const targets = await save(filepath, stream);
|
||||
|
||||
return targets[0];
|
||||
} catch (error) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
|
||||
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveProfileDetails(user) {
|
||||
const [image, description] = await Promise.all([saveProfileImage(user), saveProfileDescription(user)]);
|
||||
async function saveProfileDetails(user, args) {
|
||||
const [image, description] = await Promise.all([saveProfileImage(user, args), saveProfileDescription(user, args)]);
|
||||
|
||||
return { image, description };
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ const yaml = require('js-yaml');
|
|||
|
||||
const interpolate = require('../interpolate.js');
|
||||
|
||||
async function getIndexedPosts(user) {
|
||||
async function getIndex(user) {
|
||||
const indexFilePath = interpolate(config.library.index.file, user, null, null, false);
|
||||
|
||||
try {
|
||||
|
@ -20,4 +20,4 @@ async function getIndexedPosts(user) {
|
|||
}
|
||||
}
|
||||
|
||||
module.exports = getIndexedPosts;
|
||||
module.exports = getIndex;
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
const Promise = require('bluebird');
|
||||
|
||||
const getIndexedPosts = require('./getIndexedPosts.js');
|
||||
const getIndex = require('./getIndex.js');
|
||||
const curateUser = require('../curate/user.js');
|
||||
|
||||
const getUser = async (username, reddit) => {
|
||||
|
@ -38,9 +38,9 @@ const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
|
|||
}
|
||||
|
||||
const user = await getUser(post.author.name, reddit);
|
||||
const indexed = await getIndexedPosts(user);
|
||||
const { profile, posts: indexed } = await getIndexedPosts(user);
|
||||
|
||||
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { original: indexed, updated: [] } } };
|
||||
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { profile, original: indexed, updated: [] } } };
|
||||
}), userPosts);
|
||||
};
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ const Promise = require('bluebird');
|
|||
const fs = require('fs-extra');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
const getIndexedPosts = require('./getIndexedPosts.js');
|
||||
const getIndex = require('./getIndex.js');
|
||||
const getArchivePostIds = require('../archives/getArchivePostIds.js');
|
||||
const curateUser = require('../curate/user.js');
|
||||
const interpolate = require('../interpolate.js');
|
||||
|
@ -54,14 +54,14 @@ function getUserPostsWrap(reddit, args) {
|
|||
getPosts(username, reddit, args),
|
||||
]);
|
||||
|
||||
const { posts: indexed } = await getIndexedPosts(user);
|
||||
const { profile, posts: indexed } = await getIndex(user);
|
||||
|
||||
if (args.archives) {
|
||||
posts.push(...await getArchivedPosts(username, posts, reddit));
|
||||
}
|
||||
|
||||
if (posts.length) {
|
||||
return { ...user, posts, indexed: { original: indexed, updated: [] } };
|
||||
return { ...user, posts, indexed: { profile, original: indexed, updated: [] } };
|
||||
}
|
||||
|
||||
return null;
|
||||
|
|
Loading…
Reference in New Issue