Cleaned up entrypoint.

This commit is contained in:
ThePendulum 2018-05-05 02:27:15 +02:00
parent 9d32450864
commit 804d01d7cf
4 changed files with 110 additions and 85 deletions

View File

@ -9,21 +9,17 @@ require('promise.prototype.finally').shim();
require('array.prototype.flatten').shim(); require('array.prototype.flatten').shim();
const reddit = new snoowrap(config.reddit.api); const reddit = new snoowrap(config.reddit.api);
const args = require('./cli.js');
const curatePosts = require('./curate/posts.js'); const curatePosts = require('./curate/posts.js');
const curateUser = require('./curate/user.js');
const interpolate = require('./interpolate.js'); const interpolate = require('./interpolate.js');
const attachContentInfo = require('./fetch/info.js'); const attachContentInfo = require('./fetch/info.js');
const fetchContent = require('./fetch/content.js'); const fetchContent = require('./fetch/content.js');
const archives = require('./archives/archives.js'); const getPosts = require('./sources/getPosts.js')(reddit, args);
const getUserPosts = require('./sources/getUserPosts.js')(reddit, args);
const save = require('./save/save.js');
const saveProfileDetails = require('./save/profileDetails.js');
const args = require('./cli.js');
if(!(args.users && args.users.length) && !(args.posts && args.posts.length)) { if(!(args.users && args.users.length) && !(args.posts && args.posts.length)) {
return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user <user> or one post with --post <post-id>. See --help for more options.'); return console.log('\x1b[31m%s\x1b[0m', 'Please supply at least one user with --user <user> or one post with --post <post-id>. See --help for more options.');
@ -51,81 +47,3 @@ Promise.resolve().then(() => {
return console.error(error); return console.error(error);
}); });
function getUserPosts(usernames) {
return usernames.reduce((chain, username) => {
return chain.then(accPosts => {
return reddit.getUser(username).fetch().then(curateUser).then(saveProfileDetails).then(user => ({
user,
accPosts
}));
}).then(({user, accPosts}) => {
return reddit.getUser(username).getSubmissions({
sort: args.sort,
limit: Infinity
}).then(posts => ({
user,
accPosts: accPosts.concat(posts)
}));
}).then(({user, accPosts}) => {
if(args.archives || config.fetch.archives.search) {
return getArchivePostIds(username, accPosts.map(post => post.id)).then(postIds => {
return Promise.all(postIds.map(postId => {
return reddit.getSubmission(postId).fetch();
}));
}).then(archivedPosts => {
return {
user,
accPosts: accPosts.concat(archivedPosts)
};
});
}
return {user, accPosts};
}).then(({user, accPosts}) => {
return accPosts.map(post => Object.assign(post, {user}));
});
}, Promise.resolve([]));
};
function getPosts(postIds) {
return postIds.reduce((chain, postId) => {
return chain.then(acc => {
return reddit.getSubmission(postId).fetch().then(post => ({post, acc}));
}).then(({post, acc}) => {
if(acc.users[post.author.name]) {
return {post, acc, user: acc.users[post.author.name]}
}
if(post.author.name === '[deleted]') {
return {post, acc, user: {name: '[deleted]'}};
}
return reddit.getUser(post.author.name).fetch().then(curateUser).then(saveProfileDetails).then(user => ({post, acc, user}));
}).then(({post, acc, user}) => {
post.user = user;
acc.posts.push(post);
// keep track of users to prevent fetching one user multiple times
acc.users[user.name] = user;
return acc;
});
}, Promise.resolve({
posts: [],
users: {}
})).then(({posts, users}) => {
return posts;
});
};
function getArchivePostIds(username, exclude) {
console.log('Searching archives for posts...');
return Promise.all(config.fetch.archives.reddit.map(source => archives[source](username))).then(postIds => postIds.flatten()).then(postIds => {
return exclude ? postIds.filter(postId => !exclude.includes(postId)) : postIds;
}).then(postIds => {
console.log(`Found ${postIds.length} unique archived posts`);
return postIds;
});
};

View File

@ -0,0 +1,19 @@
'use strict';
const config = require('config');
const archives = require('./archives.js');
function getArchivePostIds(username, exclude) {
console.log('Searching archives for posts...');
return Promise.all(config.fetch.archives.reddit.map(source => archives[source](username))).then(postIds => postIds.flatten()).then(postIds => {
return exclude ? postIds.filter(postId => !exclude.includes(postId)) : postIds;
}).then(postIds => {
console.log(`Found ${postIds.length} unique archived posts`);
return postIds;
});
};
module.exports = getArchivePostIds;

41
src/sources/getPosts.js Normal file
View File

@ -0,0 +1,41 @@
'use strict';
const config = require('config');
const curateUser = require('../curate/user.js');
const saveProfileDetails = require('../save/profileDetails.js');
function getPostsWrap(reddit, args) {
return function getPosts(postIds) {
return postIds.reduce((chain, postId) => {
return chain.then(acc => {
return reddit.getSubmission(postId).fetch().then(post => ({post, acc}));
}).then(({post, acc}) => {
if(acc.users[post.author.name]) {
return {post, acc, user: acc.users[post.author.name]}
}
if(post.author.name === '[deleted]') {
return {post, acc, user: {name: '[deleted]'}};
}
return reddit.getUser(post.author.name).fetch().then(curateUser).then(saveProfileDetails).then(user => ({post, acc, user}));
}).then(({post, acc, user}) => {
post.user = user;
acc.posts.push(post);
// keep track of users to prevent fetching one user multiple times
acc.users[user.name] = user;
return acc;
});
}, Promise.resolve({
posts: [],
users: {}
})).then(({posts, users}) => {
return posts;
});
};
};
module.exports = getPostsWrap;

View File

@ -0,0 +1,47 @@
'use strict';
const config = require('config');
const getArchivePostIds = require('../archives/getArchivePostIds.js');
const curateUser = require('../curate/user.js');
const saveProfileDetails = require('../save/profileDetails.js');
function getUserPostsWrap(reddit, args) {
return function getUserPosts(usernames) {
return usernames.reduce((chain, username) => {
return chain.then(accPosts => {
return reddit.getUser(username).fetch().then(curateUser).then(saveProfileDetails).then(user => ({
user,
accPosts
}));
}).then(({user, accPosts}) => {
return reddit.getUser(username).getSubmissions({
sort: args.sort,
limit: Infinity
}).then(posts => ({
user,
accPosts: accPosts.concat(posts)
}));
}).then(({user, accPosts}) => {
if(args.archives || config.fetch.archives.search) {
return getArchivePostIds(username, accPosts.map(post => post.id)).then(postIds => {
return Promise.all(postIds.map(postId => {
return reddit.getSubmission(postId).fetch();
}));
}).then(archivedPosts => {
return {
user,
accPosts: accPosts.concat(archivedPosts)
};
});
}
return {user, accPosts};
}).then(({user, accPosts}) => {
return accPosts.map(post => Object.assign(post, {user}));
});
}, Promise.resolve([]));
};
};
module.exports = getUserPostsWrap;