Added option to index (specified) ignored posts. Saving index through save module, and it now notifies the user that an index has been written.
This commit is contained in:
parent
c1b164a5f9
commit
e48c00b529
|
@ -73,6 +73,10 @@ function getArgs() {
|
|||
describe: 'Ignore index file and force a redownload of the profile image and description',
|
||||
type: 'boolean',
|
||||
})
|
||||
.option('index-ignored', {
|
||||
describe: 'Add posts specified to be ignored to the index file, but don\'t download them',
|
||||
type: 'boolean',
|
||||
})
|
||||
.option('watch', {
|
||||
describe: 'Keep the process running and periodically check for new posts',
|
||||
type: 'boolean',
|
||||
|
|
|
@ -9,12 +9,21 @@ const hashPost = require('./hashPost.js');
|
|||
const { isAfter, isBefore, isEqual } = require('date-fns');
|
||||
|
||||
function report(curatedPosts, indexed, user, args) {
|
||||
const { posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount } = curatedPosts;
|
||||
const {
|
||||
posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored,
|
||||
} = curatedPosts;
|
||||
|
||||
if (indexedUpdated.length > 0) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`);
|
||||
}
|
||||
|
||||
if (requestedIgnored.length > 0) {
|
||||
console.log(
|
||||
'\x1b[33m%s\x1b[0m',
|
||||
`Ignoring ${requestedIgnored.length} posts because their IDs are specified to be ignored for '${user.name}'`,
|
||||
);
|
||||
}
|
||||
|
||||
if (tooOldCount > 0) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`);
|
||||
}
|
||||
|
@ -26,14 +35,14 @@ function report(curatedPosts, indexed, user, args) {
|
|||
if (beforeIndexedCount > 0) {
|
||||
console.log(
|
||||
'\x1b[33m%s\x1b[0m',
|
||||
`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`
|
||||
`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`,
|
||||
);
|
||||
}
|
||||
|
||||
if (afterIndexedCount > 0) {
|
||||
console.log(
|
||||
'\x1b[33m%s\x1b[0m',
|
||||
`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`
|
||||
`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
@ -97,12 +106,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
|
|||
const hostExcluded = args.exclude && args.exclude.includes(host.label);
|
||||
|
||||
if (ignoreIds.has(String(host.id).toLowerCase())) {
|
||||
console.log(
|
||||
'\x1b[33m%s\x1b[0m',
|
||||
`Ignoring content '${post.url}' because its ID is specified to be ignored (${permalink})`,
|
||||
);
|
||||
|
||||
return acc;
|
||||
return { ...acc, requestedIgnored: [...acc.requestedIgnored, curatedPost] };
|
||||
}
|
||||
|
||||
if (hostIncludes || hostExcluded) {
|
||||
|
@ -143,6 +147,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
|
|||
const curatedPosts = user.posts.reduce((accUserPosts, post, index) => curatePost(accUserPosts, post, user, index, indexed, ignoreIds, processed, args), {
|
||||
posts: [],
|
||||
indexedUpdated: [],
|
||||
requestedIgnored: [],
|
||||
tooOldCount: 0,
|
||||
tooRecentCount: 0,
|
||||
beforeIndexedCount: 0,
|
||||
|
@ -162,6 +167,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
|
|||
profile: user.indexed.profile,
|
||||
original: indexedOriginal,
|
||||
updated: curatedPosts.indexedUpdated,
|
||||
ignored: curatedPosts.requestedIgnored,
|
||||
oldest: indexed.oldest,
|
||||
latest: indexed.latest,
|
||||
},
|
||||
|
|
|
@ -83,7 +83,7 @@ async function fetchSaveContent(user, ep, args) {
|
|||
concurrency: config.fetch.concurrency,
|
||||
});
|
||||
|
||||
return writeToIndex(posts, profilePaths, user);
|
||||
return writeToIndex(posts, profilePaths, user, args);
|
||||
}
|
||||
|
||||
module.exports = fetchSaveContent;
|
||||
|
|
|
@ -1,16 +1,17 @@
|
|||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const fs = require('fs-extra');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
const interpolate = require('../interpolate.js');
|
||||
const interpolate = require('../interpolate');
|
||||
const textToStream = require('./textToStream');
|
||||
const save = require('./save');
|
||||
|
||||
async function writeToIndex(posts, profilePaths, user) {
|
||||
const filename = interpolate(config.library.index.file, user, null, false);
|
||||
async function writeToIndex(posts, profilePaths, user, args) {
|
||||
const filepath = interpolate(config.library.index.file, user, null, false);
|
||||
const now = new Date();
|
||||
|
||||
const newAndUpdatedEntries = posts.concat(user.indexed.updated).map((post) => {
|
||||
const newAndUpdatedEntries = posts.concat(user.indexed.updated, args.indexIgnored ? user.indexed.ignored : []).map((post) => {
|
||||
const entryPost = {
|
||||
id: post.id,
|
||||
subreddit: post.subreddit,
|
||||
|
@ -41,7 +42,7 @@ async function writeToIndex(posts, profilePaths, user) {
|
|||
return;
|
||||
}
|
||||
|
||||
return fs.writeFile(filename, yaml.safeDump(data));
|
||||
return save(filepath, textToStream(yaml.safeDump(data)));
|
||||
}
|
||||
|
||||
module.exports = writeToIndex;
|
||||
|
|
|
@ -14,7 +14,7 @@ async function getIndex(user) {
|
|||
|
||||
return yaml.safeLoad(indexFile);
|
||||
} catch (error) {
|
||||
console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`);
|
||||
console.log('\x1b[33m%s\x1b[0m', `No index file found for '${user.name}' at '${indexFilePath}'`);
|
||||
|
||||
return { profile: { image: null, description: null }, posts: [] };
|
||||
}
|
||||
|
|
|
@ -23,12 +23,12 @@ async function getUser(username, reddit) {
|
|||
|
||||
async function getPosts(username, reddit, args) {
|
||||
try {
|
||||
const user = await reddit.getUser(username).getSubmissions({
|
||||
const submissions = await reddit.getUser(username).getSubmissions({
|
||||
sort: args.sort,
|
||||
limit: Infinity,
|
||||
});
|
||||
|
||||
return user;
|
||||
return submissions;
|
||||
} catch (error) {
|
||||
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
|
||||
|
||||
|
|
Loading…
Reference in New Issue