Separated and improved request throttling.

2026-09-08 00:01:24 +02:00
parent 3e8ed79ccc
commit 8b52d73a65
5 changed files with 72 additions and 12 deletions
@@ -81,8 +81,19 @@ module.exports = {
        level: 'info',
    },
    limiter: {
+        // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
+        // to any host without its own entry in `hosts` below
        concurrency: 10,
        interval: 100,
+        hosts: {
+            // per-host overrides of `concurrency`/`interval` above, for hosts that can safely
+            // be hit harder (most media CDNs) or need to be treated more cautiously than the
+            // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
+            // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
+            // single shared limiter, each entry here runs independently: a host that's
+            // currently stalling or being throttled doesn't block any other host's downloads.
+            // 'redgifs.com': { concurrency: 20, interval: 50 },
+        },
    },
    reddit: {
        // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to