Addressed freezing indefinitely when requests are aborted.

This commit is contained in:
2026-09-02 17:27:26 +02:00
parent ba922bb385
commit 3e8ed79ccc
4 changed files with 139 additions and 48 deletions

View File

@@ -60,6 +60,13 @@ module.exports = {
avoidDuplicates: true,
retries: 3,
concurrency: 10,
// wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js).
// bhttp has no timeout of its own - a request that connects but then just stops
// responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise
// hang forever, and since the limiter's concurrency is capped, that one stuck request
// blocks every other download queued behind it indefinitely. This bounds each attempt
// so a stall fails (and retries, per `retries` above) instead of hanging forever.
timeout: 60000,
watch: {
schedule: '*/30 * * * *',
},
@@ -102,6 +109,13 @@ module.exports = {
proxy: {
use: true,
default: null,
// if a request fails (including timing out - see fetch.timeout above) and is about to
// be retried, retry it using whichever of proxied/direct is NOT what would normally
// apply for that host - e.g. the proxy itself is down or being blocked by the target
// site, or (the reverse) a host reachable directly is refusing unproxied traffic.
// Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a
// direct connection on failure is very much not what you want.
retryFlipped: false,
hosts: {
// hostnames are matched by suffix, so e.g. 'reddit.com' also covers
// 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own