DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

No longer redownload profile image and description when indexed, unless --redownload-profile is specified.

This commit is contained in:
ThePendulum 2018-07-01 03:26:26 +02:00
parent e7eef86b23
commit 1b1323dc3d
8 changed files with 37 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/app.js
View File

@ -35,7 +35,7 @@ async function getCompleteUserPosts() {
}
function fetchSavePosts(userPosts, ep) {
return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep)));
return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep, args)));
}
async function initApp() {

4
src/cli.js
View File

@ -57,6 +57,10 @@ function getArgs() {
describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
type: 'boolean',
})
.option('redownload-profile', {
describe: 'Ignore index file and force a redownload of the profile image and description',
type: 'boolean',
})
.option('watch', {
describe: 'Keep the process running and periodically check for new posts',
type: 'boolean',

1
src/curate/posts.js
View File

@ -149,6 +149,7 @@ const curatePosts = (userPosts, args) => Object.values(userPosts).reduce((accPos
...user,
posts: curatedPosts.posts,
indexed: {
profile: user.indexed.profile,
original: indexedOriginal,
updated: curatedPosts.indexedUpdated,
oldest: indexed.oldest,

4
src/fetch/content.js
View File

@ -47,8 +47,8 @@ function getFilepath(item, post, user) {
: interpolate(config.library[type], user, post, item);
}
async function fetchSaveContent(user, ep) {
const profilePaths = await saveProfileDetails(user);
async function fetchSaveContent(user, ep, args) {
const profilePaths = await saveProfileDetails(user, args);
const posts = await Promise.map(user.posts, async (post) => {
await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {

32
src/save/profileDetails.js
View File

@ -9,12 +9,18 @@ const fetchItem = require('../fetch/item.js');
const textToStream = require('./textToStream.js');
const save = require('./save.js');
async function saveProfileImage(user) {
async function saveProfileImage(user, args) {
if (!args.redownloadProfile && user.indexed.profile.description) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
return user.indexed.profile.image;
}
if (config.library.profile.image && !user.fallback && !user.deleted) {
const image = user.profile ? user.profile.image : user.image;
if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}'`);
console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
return null;
}
@ -25,45 +31,49 @@ async function saveProfileImage(user) {
});
try {
const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` })
const stream = await fetchItem(image, 0, { permalink: `https://reddit.com/user/${user.name}` });
const targets = await save(filepath, stream);
return targets[0];
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`);
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
return null;
}
}
}
async function saveProfileDescription(user) {
async function saveProfileDescription(user, args) {
if (!args.redownloadProfile && user.indexed.profile.description) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
return user.indexed.profile.description;
}
if (config.library.profile.description && !user.fallback && !user.deleted) {
if (user.profile && user.profile.description) {
const filepath = interpolate(config.library.profile.description, user);
const stream = textToStream(user.profile.description);
console.log(filepath);
try {
const targets = await save(filepath, stream);
return targets[0];
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error}`);
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
return null;
}
}
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}'`);
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
return null;
}
}
async function saveProfileDetails(user) {
const [image, description] = await Promise.all([saveProfileImage(user), saveProfileDescription(user)]);
async function saveProfileDetails(user, args) {
const [image, description] = await Promise.all([saveProfileImage(user, args), saveProfileDescription(user, args)]);
return { image, description };
}

4
src/sources/getIndexedPosts.js → src/sources/getIndex.js
View File

@ -6,7 +6,7 @@ const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
async function getIndexedPosts(user) {
async function getIndex(user) {
const indexFilePath = interpolate(config.library.index.file, user, null, null, false);
try {
@ -20,4 +20,4 @@ async function getIndexedPosts(user) {
}
}
module.exports = getIndexedPosts;
module.exports = getIndex;

6
src/sources/getPosts.js
View File

@ -2,7 +2,7 @@
const Promise = require('bluebird');
const getIndexedPosts = require('./getIndexedPosts.js');
const getIndex = require('./getIndex.js');
const curateUser = require('../curate/user.js');
const getUser = async (username, reddit) => {
@ -38,9 +38,9 @@ const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
}
const user = await getUser(post.author.name, reddit);
const indexed = await getIndexedPosts(user);
const { profile, posts: indexed } = await getIndexedPosts(user);
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { original: indexed, updated: [] } } };
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { profile, original: indexed, updated: [] } } };
}), userPosts);
};

6
src/sources/getUserPosts.js
View File

@ -5,7 +5,7 @@ const Promise = require('bluebird');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const getIndexedPosts = require('./getIndexedPosts.js');
const getIndex = require('./getIndex.js');
const getArchivePostIds = require('../archives/getArchivePostIds.js');
const curateUser = require('../curate/user.js');
const interpolate = require('../interpolate.js');
@ -54,14 +54,14 @@ function getUserPostsWrap(reddit, args) {
getPosts(username, reddit, args),
]);
const { posts: indexed } = await getIndexedPosts(user);
const { profile, posts: indexed } = await getIndex(user);
if (args.archives) {
posts.push(...await getArchivedPosts(username, posts, reddit));
}
if (posts.length) {
return { ...user, posts, indexed: { original: indexed, updated: [] } };
return { ...user, posts, indexed: { profile, original: indexed, updated: [] } };
}
return null;