Separated and improved request throttling.

config/default.js

@@ -81,8 +81,19 @@ module.exports = {
         level: 'info',
     },
     limiter: {
+        // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
+        // to any host without its own entry in `hosts` below
         concurrency: 10,
         interval: 100,
+        hosts: {
+            // per-host overrides of `concurrency`/`interval` above, for hosts that can safely
+            // be hit harder (most media CDNs) or need to be treated more cautiously than the
+            // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
+            // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
+            // single shared limiter, each entry here runs independently: a host that's
+            // currently stalling or being throttled doesn't block any other host's downloads.
+            // 'redgifs.com': { concurrency: 20, interval: 50 },
+        },
     },
     reddit: {
         // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to

src/curate/posts.js

@@ -62,6 +62,12 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
         preview: post.preview ? post.preview.images.map((image) => image.source) : null,
         host,
         direct: post.direct,
+        // reddit's submission listing (src/reddit/client.js's getSubmissions, fetched once up
+        // front) already embeds these for gallery posts - carrying them through lets
+        // methods/redditAlbum.js use them directly instead of re-fetching the same post
+        // individually through reddit's own separately-throttled client (see there)
+        mediaMetadata: post.media_metadata || null,
+        galleryData: post.gallery_data || null,
         // count only - the full thread (when needed) is fetched separately via methods/self.js
         comments: post.num_comments,
         hash: hashPost(post),

src/fetch/item.js

@@ -8,7 +8,7 @@ const phash = require('sharp-phash');
 
 const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
-const limiter = require('../limiter').items;
+const limiterFor = require('../limiter');
 
 // bhttp has no timeout at all by default - if the server (or, more likely, a proxy tunnel
 // that connects fine but then never relays anything) just stops responding mid-request without
@@ -42,7 +42,7 @@ async function fetchItem(url, attempt, context) {
         // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
         const flip = config.proxy.retryFlipped && attempt > 0;
 
-        const res = await limiter.schedule(async () => withTimeout(
+        const res = await limiterFor(url).schedule(async () => withTimeout(
             bhttp.get(url, { headers: context.headers, agent: agentFor(url, { flip }) }),
             config.fetch.timeout,
             url,

src/limiter.js

@@ -1,13 +1,50 @@
 'use strict';
 
 const Bottleneck = require('bottleneck');
+const config = require('config');
 
 const args = require('./cli')();
 
-module.exports = {
+// same suffix matching as config.proxy.hosts (src/proxy.js): a bare hostname also matches its
-    // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle)
+// subdomains, so 'redgifs.com' catches 'media.redgifs.com' but not the reverse
-    items: new Bottleneck({
+function hostMatches(hostname, pattern) {
-        maxConcurrent: args.concurrency,
+    const normalized = pattern.replace(/^\./, '').toLowerCase();
-        minTime: args.interval,
+
-    }),
+    return hostname === normalized || hostname.endsWith(`.${normalized}`);
-};
+}
+
+// reddit's own API requests are throttled separately, inside src/reddit/client.js
+// (config.reddit.throttle) - this only governs item/media downloads (src/fetch/item.js)
+const defaultLimiter = new Bottleneck({
+    maxConcurrent: args.concurrency,
+    minTime: args.interval,
+});
+
+const hostLimiters = new Map();
+
+// config.limiter.hosts: most media CDNs (redgifs, imgur, i.redd.it, ...) don't need to be
+// treated as cautiously as reddit's own API, but a single shared limiter forces every host
+// down to whatever pace the most conservative one requires - worse, it means one host that's
+// currently stalling/throttling (see src/fetch/item.js's timeout handling) occupies the one
+// shared slot and blocks every *other* host's downloads behind it too, even though they have
+// nothing to do with each other. Giving a host its own limiter here lets it run at its own
+// concurrency/interval, independently of every other host's.
+function limiterFor(url) {
+    const { hostname } = new URL(url);
+    const hosts = config.limiter?.hosts || {};
+    const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern));
+
+    if (!matchedPattern) {
+        return defaultLimiter;
+    }
+
+    if (!hostLimiters.has(matchedPattern)) {
+        const { concurrency = args.concurrency, interval = args.interval } = hosts[matchedPattern];
+
+        hostLimiters.set(matchedPattern, new Bottleneck({ maxConcurrent: concurrency, minTime: interval }));
+    }
+
+    return hostLimiters.get(matchedPattern);
+}
+
+module.exports = limiterFor;

src/methods/redditAlbum.js

@@ -17,9 +17,15 @@ function normalizeType(type) {
 }
 
 async function redditAlbum(host, post, { reddit }) {
-    // fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
+    // reddit's submission listing already embeds media_metadata/gallery_data for gallery posts,
-    // reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    // and curate/posts.js carries them through as post.mediaMetadata/post.galleryData - reuse
-    const data = await reddit.getSubmission(post.id);
+    // those instead of re-fetching the same post individually through reddit's own client
+    // (reservoir: 6/min, maxConcurrent: 1 - see src/reddit/client.js), which would otherwise cost
+    // one full throttled round trip per gallery post. Only fall back to a fresh fetch if the
+    // listing didn't have it for some reason (e.g. gallery_data null on the listing too).
+    const data = post.mediaMetadata
+        ? { media_metadata: post.mediaMetadata, gallery_data: post.galleryData }
+        : await reddit.getSubmission(post.id);
     const mediaMetadata = data.media_metadata || {};
 
     // gallery_data (the ordered {media_id, is_deleted, ...} list for this gallery) has been