From edf77e259b15abead1e4ed9915e56c544f8cfeb0 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Tue, 22 May 2018 22:55:50 +0200 Subject: [PATCH] Refactored post fetching. Cleaner code that will now fetch archived posts even if the user no longer exists on reddit. Expanded gfycat URL detection. --- src/app.js | 1 - src/cli.js | 3 +- src/curate/user.js | 1 + src/dissectLink.js | 2 +- src/interpolate.js | 2 +- src/save/profileDetails.js | 6 +-- src/sources/getUserPosts.js | 76 ++++++++++++++++++++----------------- 7 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/app.js b/src/app.js index b1a8ac5..43fdab8 100644 --- a/src/app.js +++ b/src/app.js @@ -46,4 +46,3 @@ Promise.resolve().then(() => { }).catch(error => { return console.error(error); }); - diff --git a/src/cli.js b/src/cli.js index 470b15f..fcf8b2f 100644 --- a/src/cli.js +++ b/src/cli.js @@ -31,5 +31,6 @@ module.exports = yargs.command('npm start -- --user ').option('users', type: 'array' }).option('archives', { describe: 'Search archives for deleted posts', - type: 'boolean' + type: 'boolean', + default: config.fetch.archives.search }).argv; diff --git a/src/curate/user.js b/src/curate/user.js index c9a1ff6..7c8ed6d 100644 --- a/src/curate/user.js +++ b/src/curate/user.js @@ -11,6 +11,7 @@ function curateUser(user) { gold: user.is_gold, verified: user.verified, verifiedEmail: user.has_verified_email, + fallback: false }; if(user.subreddit) { diff --git a/src/dissectLink.js b/src/dissectLink.js index 37c99bb..1a05bcb 100644 --- a/src/dissectLink.js +++ b/src/dissectLink.js @@ -37,7 +37,7 @@ const hosts = [{ }, { method: 'gfycat', label: 'gfycat', - pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/:id(.:ext)') + pattern: new urlPattern('http(s)\\://(:server.)gfycat.com/(gifs/detail/):id(.:ext)') }, { method: 'eroshareAlbum', label: 'eroshare', diff --git a/src/interpolate.js b/src/interpolate.js index ebc5b3f..2a0d6aa 100644 --- a/src/interpolate.js +++ b/src/interpolate.js @@ -38,7 +38,7 @@ function interpolate(pattern, user, post, item) { Object.assign(vars, { $postId: post.id, $postTitle: (post.title || '').slice(0, config.library.titleLength), - $postUser: post.user || user.user, + $postUser: post.user, $postDate: dateFns.format(post.datetime, dateFormat), $postIndex: post.index + config.library.indexOffset, $subreddit: post.subreddit, diff --git a/src/save/profileDetails.js b/src/save/profileDetails.js index c42d0ca..320a97f 100644 --- a/src/save/profileDetails.js +++ b/src/save/profileDetails.js @@ -9,7 +9,7 @@ const textToStream = require('./textToStream.js'); const save = require('./save.js'); function saveProfileDetails(user) { - if(config.library.profile.image) { + if(config.library.profile.image && !user.fallback) { const image = user.profile ? user.profile.image : user.image; if(config.library.profile.avoidAvatar && new urlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) { @@ -20,13 +20,13 @@ function saveProfileDetails(user) { url: image }); - fetchItem(image).then(stream => save(filepath, stream)).catch(error => { + fetchItem(image, 0, {permalink: `https://reddit.com/user/${user.name}`}).then(stream => save(filepath, stream)).catch(error => { console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error}`); }); } } - if(config.library.profile.description) { + if(config.library.profile.description && !user.fallback) { if(user.profile && user.profile.description) { const filepath = interpolate(config.library.profile.description, user); const stream = textToStream(user.profile.description); diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 57f24c1..65c274d 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -6,42 +6,48 @@ const getArchivePostIds = require('../archives/getArchivePostIds.js'); const curateUser = require('../curate/user.js'); const saveProfileDetails = require('../save/profileDetails.js'); -function getUserPostsWrap(reddit, args) { - return function getUserPosts(usernames) { - return usernames.reduce((chain, username) => { - return chain.then(accPosts => { - return reddit.getUser(username).fetch().then(curateUser).then(saveProfileDetails).then(user => ({ - user, - accPosts - })); - }).then(({user, accPosts}) => { - return reddit.getUser(username).getSubmissions({ - sort: args.sort, - limit: Infinity - }).then(posts => ({ - user, - accPosts: accPosts.concat(posts) - })); - }).then(({user, accPosts}) => { - if(args.archives || config.fetch.archives.search) { - return getArchivePostIds(username, accPosts.map(post => post.id)).then(postIds => { - return Promise.all(postIds.map(postId => { - return reddit.getSubmission(postId).fetch(); - })); - }).then(archivedPosts => { - return { - user, - accPosts: accPosts.concat(archivedPosts) - }; - }); - } +const getUser = (username, reddit) => { + return reddit.getUser(username).fetch().then(user => curateUser(user)).catch(error => { + console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); - return {user, accPosts}; - }).then(({user, accPosts}) => { - return accPosts.map(post => Object.assign(post, {user})); - }); - }, Promise.resolve([])); - }; + return { + name: username, + fallback: true + }; + }); }; +const getPosts = (username, reddit, args) => { + return reddit.getUser(username).getSubmissions({ + sort: args.sort, + limit: Infinity + }).catch(error => { + console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); + + return []; + }); +}; + +const getUserPostsWrap = (reddit, args) => usernames => Promise.all( + usernames.map(async username => { + const [user, posts] = await Promise.all([ + getUser(username, reddit), + getPosts(username, reddit, args) + ]); + + if(user) { + saveProfileDetails(user); + } + + if(args.archives) { + const postIds = await getArchivePostIds(username, posts.map(post => post.id)); + const archivedPosts = await Promise.all(postIds.map(postId => reddit.getSubmission(postId).fetch())); + + posts.push(...archivedPosts); + } + + return posts.map(post => Object.assign(post, {user})); + }) +).then(posts => posts.flatten()); + module.exports = getUserPostsWrap;