Fixed index file for single post fetching.

This commit is contained in:
ThePendulum 2018-07-01 01:07:32 +02:00
parent 240146e554
commit c639503667
6 changed files with 52 additions and 38 deletions

View File

@ -19,7 +19,11 @@ const getPosts = require('./sources/getPosts.js')(reddit, args);
const getUserPosts = require('./sources/getUserPosts.js')(reddit, args);
async function getCompleteUserPosts() {
let userPosts = await getUserPosts(args.users);
let userPosts = {};
if (args.users) {
userPosts = await getUserPosts(args.users);
}
if (args.posts) {
userPosts = await getPosts(args.posts, userPosts);

View File

@ -1,7 +1,5 @@
'use strict';
const path = require('path');
function curateUser(user) {
const curatedUser = {
id: user.id,

View File

@ -7,6 +7,8 @@ const ffmpeg = require('fluent-ffmpeg');
function save(requestedFilepath, streams, item, post) {
const filepath = requestedFilepath.split('/').map(component => {
console.log(component);
if(config.library.truncate && component.length > config.library.truncate.limit) {
return component.slice(0, config.library.truncate.limit - config.library.truncate.truncator.length) + config.library.truncate.truncator;
}
@ -14,6 +16,8 @@ function save(requestedFilepath, streams, item, post) {
return component;
}).join(path.sep);
console.log(filepath);
const pathComponents = path.parse(filepath);
// allow for single stream argument

View File

@ -0,0 +1,23 @@
'use strict';
const config = require('config');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
async function getIndexedPosts(user) {
const indexFilePath = interpolate(config.library.index.file, user, null, null, false);
try {
const indexFile = await fs.readFile(indexFilePath, 'utf8');
return yaml.safeLoad(indexFile);
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`);
return [];
}
}
module.exports = getIndexedPosts;

View File

@ -1,49 +1,47 @@
'use strict';
const Promise = require('bluebird');
const config = require('config');
const getIndexedPosts = require('./getIndexedPosts.js');
const curateUser = require('../curate/user.js');
const saveProfileDetails = require('../save/profileDetails.js');
const getUser = async (username, reddit) => {
try {
const user = await reddit.getUser(username).fetch();
return curateUser(user);
} catch(error) {
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return {
name: username,
fallback: true
fallback: true,
};
}
};
const getPostsWrap = (reddit, args) => {
return function getPosts(postIds, userPosts = {}) {
const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
const post = await reddit.getSubmission(postId).fetch();
post.direct = true;
if(accUserPosts[post.author.name]) {
if (accUserPosts[post.author.name]) {
accUserPosts[post.author.name].posts = accUserPosts[post.author.name].posts.concat(post);
return accUserPosts;
}
// don't attempt to fetch deleted user
if(post.author.name === '[deleted]') {
return {...accUserPosts, '[deleted]': {name: '[deleted]', deleted: true, posts: [post]}};
if (post.author.name === '[deleted]') {
return { ...accUserPosts, '[deleted]': { name: '[deleted]', deleted: true, posts: [post] } };
}
const user = await getUser(post.author.name, reddit);
const indexed = await getIndexedPosts(user);
return {...accUserPosts, [post.author.name]: {...user, posts: [post]}}
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { original: indexed, updated: [] } } };
}), userPosts);
};
};
module.exports = getPostsWrap;

View File

@ -5,6 +5,7 @@ const Promise = require('bluebird');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const getIndexedPosts = require('./getIndexedPosts.js');
const getArchivePostIds = require('../archives/getArchivePostIds.js');
const curateUser = require('../curate/user.js');
const interpolate = require('../interpolate.js');
@ -45,20 +46,6 @@ async function getArchivedPosts(username, posts, reddit) {
return Promise.all(postIds.map(postId => reddit.getSubmission(postId).fetch()));
}
async function getIndexedPosts(user) {
const indexFilePath = interpolate(config.library.index.file, user, null, null, false);
try {
const indexFile = await fs.readFile(indexFilePath, 'utf8');
return yaml.safeLoad(indexFile);
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`);
return [];
}
}
function getUserPostsWrap(reddit, args) {
return async function getUserPosts(usernames) {
const users = await Promise.map(usernames, async (username) => {