Addressed freezing indefinitely when requests are aborted.

src/fetch/item.js

@@ -10,6 +10,21 @@ const { agentFor } = require('../proxy');
 const logger = require('../logger')(__filename);
 const limiter = require('../limiter').items;
 
+// bhttp has no timeout at all by default - if the server (or, more likely, a proxy tunnel
+// that connects fine but then never relays anything) just stops responding mid-request without
+// erroring, the returned promise never settles. With the limiter's concurrency capped, that one
+// stuck request blocks every other download queued behind it, indefinitely. Race it against a
+// hard wall-clock ceiling so a stall fails (and retries, below) instead of hanging forever.
+function withTimeout(promise, ms, url) {
+    let timer;
+
+    const timedOut = new Promise((resolve, reject) => {
+        timer = setTimeout(() => reject(new Error(`Timed out after ${ms}ms fetching '${url}'`)), ms);
+    });
+
+    return Promise.race([promise, timedOut]).finally(() => clearTimeout(timer));
+}
+
 async function fetchItem(url, attempt, context) {
     async function retry(error) {
         logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${context.post ? context.post.permalink : 'no post'})`);
@@ -23,7 +38,15 @@ async function fetchItem(url, attempt, context) {
 
     try {
         // throw new Error('Failed it!');
-        const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) }));
+        // config.proxy.retryFlipped: once this url has already failed once, retry it via
+        // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
+        const flip = config.proxy.retryFlipped && attempt > 0;
+
+        const res = await limiter.schedule(async () => withTimeout(
+            bhttp.get(url, { headers: context.headers, agent: agentFor(url, { flip }) }),
+            config.fetch.timeout,
+            url,
+        ));
 
         if (res.statusCode !== 200) {
             throw new Error(`Response not OK for ${url} (${res.statusCode})`);