From 99c7d143f7f00ca3cd61bdbec3e1e3b749534508 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:56 +0200 Subject: [PATCH] Fixed index file for single post fetching. --- src/app.js | 6 ++++- src/curate/user.js | 2 -- src/save/save.js | 4 ++++ src/sources/getIndexedPosts.js | 23 +++++++++++++++++++ src/sources/getPosts.js | 40 ++++++++++++++++------------------ src/sources/getUserPosts.js | 15 +------------ 6 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 src/sources/getIndexedPosts.js diff --git a/src/app.js b/src/app.js index 327eb53..9b02127 100644 --- a/src/app.js +++ b/src/app.js @@ -19,7 +19,11 @@ const getPosts = require('./sources/getPosts.js')(reddit, args); const getUserPosts = require('./sources/getUserPosts.js')(reddit, args); async function getCompleteUserPosts() { - let userPosts = await getUserPosts(args.users); + let userPosts = {}; + + if (args.users) { + userPosts = await getUserPosts(args.users); + } if (args.posts) { userPosts = await getPosts(args.posts, userPosts); diff --git a/src/curate/user.js b/src/curate/user.js index 7c8ed6d..b51b512 100644 --- a/src/curate/user.js +++ b/src/curate/user.js @@ -1,7 +1,5 @@ 'use strict'; -const path = require('path'); - function curateUser(user) { const curatedUser = { id: user.id, diff --git a/src/save/save.js b/src/save/save.js index 42fe12c..2d945aa 100644 --- a/src/save/save.js +++ b/src/save/save.js @@ -7,6 +7,8 @@ const ffmpeg = require('fluent-ffmpeg'); function save(requestedFilepath, streams, item, post) { const filepath = requestedFilepath.split('/').map(component => { + console.log(component); + if(config.library.truncate && component.length > config.library.truncate.limit) { return component.slice(0, config.library.truncate.limit - config.library.truncate.truncator.length) + config.library.truncate.truncator; } @@ -14,6 +16,8 @@ function save(requestedFilepath, streams, item, post) { return component; }).join(path.sep); + console.log(filepath); + const pathComponents = path.parse(filepath); // allow for single stream argument diff --git a/src/sources/getIndexedPosts.js b/src/sources/getIndexedPosts.js new file mode 100644 index 0000000..150c32f --- /dev/null +++ b/src/sources/getIndexedPosts.js @@ -0,0 +1,23 @@ +'use strict'; + +const config = require('config'); +const fs = require('fs-extra'); +const yaml = require('js-yaml'); + +const interpolate = require('../interpolate.js'); + +async function getIndexedPosts(user) { + const indexFilePath = interpolate(config.library.index.file, user, null, null, false); + + try { + const indexFile = await fs.readFile(indexFilePath, 'utf8'); + + return yaml.safeLoad(indexFile); + } catch (error) { + console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`); + + return []; + } +} + +module.exports = getIndexedPosts; diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index e257583..db201c3 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -1,49 +1,47 @@ 'use strict'; const Promise = require('bluebird'); -const config = require('config'); +const getIndexedPosts = require('./getIndexedPosts.js'); const curateUser = require('../curate/user.js'); -const saveProfileDetails = require('../save/profileDetails.js'); const getUser = async (username, reddit) => { try { const user = await reddit.getUser(username).fetch(); return curateUser(user); - } catch(error) { + } catch (error) { console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); return { name: username, - fallback: true + fallback: true, }; } }; -const getPostsWrap = (reddit, args) => { - return function getPosts(postIds, userPosts = {}) { - return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { - const post = await reddit.getSubmission(postId).fetch(); +const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { + return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { + const post = await reddit.getSubmission(postId).fetch(); - post.direct = true; + post.direct = true; - if(accUserPosts[post.author.name]) { - accUserPosts[post.author.name].posts = accUserPosts[post.author.name].posts.concat(post); + if (accUserPosts[post.author.name]) { + accUserPosts[post.author.name].posts = accUserPosts[post.author.name].posts.concat(post); - return accUserPosts; - } + return accUserPosts; + } - // don't attempt to fetch deleted user - if(post.author.name === '[deleted]') { - return {...accUserPosts, '[deleted]': {name: '[deleted]', deleted: true, posts: [post]}}; - } + // don't attempt to fetch deleted user + if (post.author.name === '[deleted]') { + return { ...accUserPosts, '[deleted]': { name: '[deleted]', deleted: true, posts: [post] } }; + } - const user = await getUser(post.author.name, reddit); + const user = await getUser(post.author.name, reddit); + const indexed = await getIndexedPosts(user); - return {...accUserPosts, [post.author.name]: {...user, posts: [post]}} - }), userPosts); - }; + return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { original: indexed, updated: [] } } }; + }), userPosts); }; module.exports = getPostsWrap; diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index 1fa53fa..e469e37 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -5,6 +5,7 @@ const Promise = require('bluebird'); const fs = require('fs-extra'); const yaml = require('js-yaml'); +const getIndexedPosts = require('./getIndexedPosts.js'); const getArchivePostIds = require('../archives/getArchivePostIds.js'); const curateUser = require('../curate/user.js'); const interpolate = require('../interpolate.js'); @@ -45,20 +46,6 @@ async function getArchivedPosts(username, posts, reddit) { return Promise.all(postIds.map(postId => reddit.getSubmission(postId).fetch())); } -async function getIndexedPosts(user) { - const indexFilePath = interpolate(config.library.index.file, user, null, null, false); - - try { - const indexFile = await fs.readFile(indexFilePath, 'utf8'); - - return yaml.safeLoad(indexFile); - } catch (error) { - console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`); - - return []; - } -} - function getUserPostsWrap(reddit, args) { return async function getUserPosts(usernames) { const users = await Promise.map(usernames, async (username) => {