diff --git a/config/default.js b/config/default.js index 471e927..37355f5 100644 --- a/config/default.js +++ b/config/default.js @@ -45,6 +45,7 @@ module.exports = { limit: 1000, avoidDuplicates: true, retries: 3, + concurrency: 10, watch: { schedule: '*/30 * * * *', }, diff --git a/src/app.js b/src/app.js index 057bad1..49f8b69 100644 --- a/src/app.js +++ b/src/app.js @@ -3,6 +3,7 @@ const config = require('config'); const Snoowrap = require('snoowrap'); const fs = require('fs-extra'); +const Promise = require('bluebird'); const exiftool = require('node-exiftool'); const exiftoolBin = require('dist-exiftool'); const cron = require('node-cron'); @@ -69,7 +70,8 @@ async function getCompleteUserPosts() { } function fetchSavePosts(userPosts, ep) { - return Promise.all(Object.values(userPosts).map(user => fetchSaveContent(user, ep, args))); + // don't map to apply concurrency limit and reduce network stress + return Promise.reduce(Object.values(userPosts), (acc, user) => fetchSaveContent(user, ep, args), null); } async function initApp() { diff --git a/src/curate/posts.js b/src/curate/posts.js index 4518d37..6f16d75 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -36,7 +36,7 @@ function report(curatedPosts, indexed, user, args) { `Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'` ); } -}; +} function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) { const host = dissectLink(post.url); diff --git a/src/fetch/content.js b/src/fetch/content.js index 2a710e0..07adb00 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -73,6 +73,8 @@ async function fetchSaveContent(user, ep, args) { }, []); return post; + }, { + concurrency: config.fetch.concurrency, }); return writeToIndex(posts, profilePaths, user); diff --git a/src/save/writeToIndex.js b/src/save/writeToIndex.js index ce2f5a7..784c664 100644 --- a/src/save/writeToIndex.js +++ b/src/save/writeToIndex.js @@ -10,24 +10,6 @@ async function writeToIndex(posts, profilePaths, user) { const filename = interpolate(config.library.index.file, user, null, false); const now = new Date(); - /* - // Individual posts are wrapped in [] to get a YAML array value for each individual item, allowing them to be joined manually with a newline - // between each entry to improve human readability of the index while maintaining a valid YAML list - const originalEntries = user.indexed.original.map(entry => yaml.safeDump([entry])); - const newAndUpdatedEntries = posts.concat(user.indexed.updated).map(post => yaml.safeDump([{ - id: post.id, - subreddit: post.subreddit, - permalink: post.permalink, - url: post.url, - hostId: post.host.id, - date: post.datetime, - indexed: now, - title: post.title, - }])); - - const entries = newAndUpdatedEntries.concat(originalEntries).join('\n'); - */ - const newAndUpdatedEntries = posts.concat(user.indexed.updated).map(post => ({ id: post.id, subreddit: post.subreddit,