DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added option to index (specified) ignored posts. Saving index through save module, and it now notifies the user that an index has been written.

This commit is contained in:
ThePendulum 2018-07-07 02:17:41 +02:00
parent fd4fe8a0c5
commit d9ce4dd056
6 changed files with 30 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli.js
View File

@ -73,6 +73,10 @@ function getArgs() {
describe: 'Ignore index file and force a redownload of the profile image and description',
type: 'boolean',
})
.option('index-ignored', {
describe: 'Add posts specified to be ignored to the index file, but don\'t download them',
type: 'boolean',
})
.option('watch', {
describe: 'Keep the process running and periodically check for new posts',
type: 'boolean',

24
src/curate/posts.js
View File

@ -9,12 +9,21 @@ const hashPost = require('./hashPost.js');
const { isAfter, isBefore, isEqual } = require('date-fns');
function report(curatedPosts, indexed, user, args) {
const { posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount } = curatedPosts;
const {
posts, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored,
} = curatedPosts;
if (indexedUpdated.length > 0) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`);
}
if (requestedIgnored.length > 0) {
console.log(
'\x1b[33m%s\x1b[0m',
`Ignoring ${requestedIgnored.length} posts because their IDs are specified to be ignored for '${user.name}'`,
);
}
if (tooOldCount > 0) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`);
}
@ -26,14 +35,14 @@ function report(curatedPosts, indexed, user, args) {
if (beforeIndexedCount > 0) {
console.log(
'\x1b[33m%s\x1b[0m',
`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`
`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`,
);
}
if (afterIndexedCount > 0) {
console.log(
'\x1b[33m%s\x1b[0m',
`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`
`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`,
);
}
}
@ -97,12 +106,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
const hostExcluded = args.exclude && args.exclude.includes(host.label);
if (ignoreIds.has(String(host.id).toLowerCase())) {
console.log(
'\x1b[33m%s\x1b[0m',
`Ignoring content '${post.url}' because its ID is specified to be ignored (${permalink})`,
);
return acc;
return { ...acc, requestedIgnored: [...acc.requestedIgnored, curatedPost] };
}
if (hostIncludes || hostExcluded) {
@ -143,6 +147,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
const curatedPosts = user.posts.reduce((accUserPosts, post, index) => curatePost(accUserPosts, post, user, index, indexed, ignoreIds, processed, args), {
posts: [],
indexedUpdated: [],
requestedIgnored: [],
tooOldCount: 0,
tooRecentCount: 0,
beforeIndexedCount: 0,
@ -162,6 +167,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
profile: user.indexed.profile,
original: indexedOriginal,
updated: curatedPosts.indexedUpdated,
ignored: curatedPosts.requestedIgnored,
oldest: indexed.oldest,
latest: indexed.latest,
},

2
src/fetch/content.js
View File

@ -83,7 +83,7 @@ async function fetchSaveContent(user, ep, args) {
concurrency: config.fetch.concurrency,
});
return writeToIndex(posts, profilePaths, user);
return writeToIndex(posts, profilePaths, user, args);
}
module.exports = fetchSaveContent;

13
src/save/writeToIndex.js
View File

@ -1,16 +1,17 @@
'use strict';
const config = require('config');
const fs = require('fs-extra');
const yaml = require('js-yaml');
const interpolate = require('../interpolate.js');
const interpolate = require('../interpolate');
const textToStream = require('./textToStream');
const save = require('./save');
async function writeToIndex(posts, profilePaths, user) {
const filename = interpolate(config.library.index.file, user, null, false);
async function writeToIndex(posts, profilePaths, user, args) {
const filepath = interpolate(config.library.index.file, user, null, false);
const now = new Date();
const newAndUpdatedEntries = posts.concat(user.indexed.updated).map((post) => {
const newAndUpdatedEntries = posts.concat(user.indexed.updated, args.indexIgnored ? user.indexed.ignored : []).map((post) => {
const entryPost = {
id: post.id,
subreddit: post.subreddit,
@ -41,7 +42,7 @@ async function writeToIndex(posts, profilePaths, user) {
return;
}
return fs.writeFile(filename, yaml.safeDump(data));
return save(filepath, textToStream(yaml.safeDump(data)));
}
module.exports = writeToIndex;

2
src/sources/getIndex.js
View File

@ -14,7 +14,7 @@ async function getIndex(user) {
return yaml.safeLoad(indexFile);
} catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not load index file for '${user.name}' at '${indexFilePath}': ${error}`);
console.log('\x1b[33m%s\x1b[0m', `No index file found for '${user.name}' at '${indexFilePath}'`);
return { profile: { image: null, description: null }, posts: [] };
}

4
src/sources/getUserPosts.js
View File

@ -23,12 +23,12 @@ async function getUser(username, reddit) {
async function getPosts(username, reddit, args) {
try {
const user = await reddit.getUser(username).getSubmissions({
const submissions = await reddit.getUser(username).getSubmissions({
sort: args.sort,
limit: Infinity,
});
return user;
return submissions;
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);