Using new HTTP module with a dynamic rate limiter.

2020-11-22 04:07:09 +01:00
parent 5d0fe44130
commit b9b777c621
27 changed files with 358 additions and 175 deletions
--- a/config/default.js
+++ b/config/default.js
@@ -197,6 +197,12 @@ module.exports = {
 			'www.deeper.com',
 		],
 	},
+	limits: {
+		default: {
+			interval: 50,
+			concurrency: 20,
+		},
+	},
 	fetchAfter: [1, 'week'],
 	missingDateLimit: 3,
 	media: {