From 8b52d73a65f39273f8025625d3eca435eda664ea Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 8 Sep 2026 00:01:24 +0200 Subject: [PATCH] Separated and improved request throttling. --- config/default.js | 11 ++++++++ src/curate/posts.js | 6 +++++ src/fetch/item.js | 4 +-- src/limiter.js | 51 ++++++++++++++++++++++++++++++++------ src/methods/redditAlbum.js | 12 ++++++--- 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/config/default.js b/config/default.js index c707bf5..274e792 100644 --- a/config/default.js +++ b/config/default.js @@ -81,8 +81,19 @@ module.exports = { level: 'info', }, limiter: { + // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies + // to any host without its own entry in `hosts` below concurrency: 10, interval: 100, + hosts: { + // per-host overrides of `concurrency`/`interval` above, for hosts that can safely + // be hit harder (most media CDNs) or need to be treated more cautiously than the + // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare + // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a + // single shared limiter, each entry here runs independently: a host that's + // currently stalling or being throttled doesn't block any other host's downloads. + // 'redgifs.com': { concurrency: 20, interval: 50 }, + }, }, reddit: { // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to diff --git a/src/curate/posts.js b/src/curate/posts.js index 02ddc96..7680c91 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -62,6 +62,12 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) preview: post.preview ? post.preview.images.map((image) => image.source) : null, host, direct: post.direct, + // reddit's submission listing (src/reddit/client.js's getSubmissions, fetched once up + // front) already embeds these for gallery posts - carrying them through lets + // methods/redditAlbum.js use them directly instead of re-fetching the same post + // individually through reddit's own separately-throttled client (see there) + mediaMetadata: post.media_metadata || null, + galleryData: post.gallery_data || null, // count only - the full thread (when needed) is fetched separately via methods/self.js comments: post.num_comments, hash: hashPost(post), diff --git a/src/fetch/item.js b/src/fetch/item.js index 882af78..65631b4 100644 --- a/src/fetch/item.js +++ b/src/fetch/item.js @@ -8,7 +8,7 @@ const phash = require('sharp-phash'); const { agentFor } = require('../proxy'); const logger = require('../logger')(__filename); -const limiter = require('../limiter').items; +const limiterFor = require('../limiter'); // bhttp has no timeout at all by default - if the server (or, more likely, a proxy tunnel // that connects fine but then never relays anything) just stops responding mid-request without @@ -42,7 +42,7 @@ async function fetchItem(url, attempt, context) { // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl() const flip = config.proxy.retryFlipped && attempt > 0; - const res = await limiter.schedule(async () => withTimeout( + const res = await limiterFor(url).schedule(async () => withTimeout( bhttp.get(url, { headers: context.headers, agent: agentFor(url, { flip }) }), config.fetch.timeout, url, diff --git a/src/limiter.js b/src/limiter.js index ba21479..45d2f47 100644 --- a/src/limiter.js +++ b/src/limiter.js @@ -1,13 +1,50 @@ 'use strict'; const Bottleneck = require('bottleneck'); +const config = require('config'); const args = require('./cli')(); -module.exports = { - // reddit requests are throttled inside src/reddit/client.js itself (config.reddit.throttle) - items: new Bottleneck({ - maxConcurrent: args.concurrency, - minTime: args.interval, - }), -}; +// same suffix matching as config.proxy.hosts (src/proxy.js): a bare hostname also matches its +// subdomains, so 'redgifs.com' catches 'media.redgifs.com' but not the reverse +function hostMatches(hostname, pattern) { + const normalized = pattern.replace(/^\./, '').toLowerCase(); + + return hostname === normalized || hostname.endsWith(`.${normalized}`); +} + +// reddit's own API requests are throttled separately, inside src/reddit/client.js +// (config.reddit.throttle) - this only governs item/media downloads (src/fetch/item.js) +const defaultLimiter = new Bottleneck({ + maxConcurrent: args.concurrency, + minTime: args.interval, +}); + +const hostLimiters = new Map(); + +// config.limiter.hosts: most media CDNs (redgifs, imgur, i.redd.it, ...) don't need to be +// treated as cautiously as reddit's own API, but a single shared limiter forces every host +// down to whatever pace the most conservative one requires - worse, it means one host that's +// currently stalling/throttling (see src/fetch/item.js's timeout handling) occupies the one +// shared slot and blocks every *other* host's downloads behind it too, even though they have +// nothing to do with each other. Giving a host its own limiter here lets it run at its own +// concurrency/interval, independently of every other host's. +function limiterFor(url) { + const { hostname } = new URL(url); + const hosts = config.limiter?.hosts || {}; + const matchedPattern = Object.keys(hosts).find((pattern) => hostMatches(hostname, pattern)); + + if (!matchedPattern) { + return defaultLimiter; + } + + if (!hostLimiters.has(matchedPattern)) { + const { concurrency = args.concurrency, interval = args.interval } = hosts[matchedPattern]; + + hostLimiters.set(matchedPattern, new Bottleneck({ maxConcurrent: concurrency, minTime: interval })); + } + + return hostLimiters.get(matchedPattern); +} + +module.exports = limiterFor; diff --git a/src/methods/redditAlbum.js b/src/methods/redditAlbum.js index 8694d35..b9be009 100644 --- a/src/methods/redditAlbum.js +++ b/src/methods/redditAlbum.js @@ -17,9 +17,15 @@ function normalizeType(type) { } async function redditAlbum(host, post, { reddit }) { - // fetched through the authenticated/throttled client, not a raw fetch()/bhttp request - - // reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests - const data = await reddit.getSubmission(post.id); + // reddit's submission listing already embeds media_metadata/gallery_data for gallery posts, + // and curate/posts.js carries them through as post.mediaMetadata/post.galleryData - reuse + // those instead of re-fetching the same post individually through reddit's own client + // (reservoir: 6/min, maxConcurrent: 1 - see src/reddit/client.js), which would otherwise cost + // one full throttled round trip per gallery post. Only fall back to a fresh fetch if the + // listing didn't have it for some reason (e.g. gallery_data null on the listing too). + const data = post.mediaMetadata + ? { media_metadata: post.mediaMetadata, gallery_data: post.galleryData } + : await reddit.getSubmission(post.id); const mediaMetadata = data.media_metadata || {}; // gallery_data (the ordered {media_id, is_deleted, ...} list for this gallery) has been