'use strict'; const config = require('config'); const bhttp = require('bhttp'); const blake2 = require('blake2'); const phash = require('sharp-phash'); // const phashDistance = require('sharp-phash/distance'); const { agentFor } = require('../proxy'); const logger = require('../logger')(__filename); const limiterFor = require('../limiter'); async function fetchItem(url, attempt, context) { async function retry(error) { logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${context.post ? context.post.permalink : 'no post'})`); if (attempt < config.fetch.retries) { return fetchItem(url, attempt + 1, context); } return null; } try { // throw new Error('Failed it!'); // config.proxy.retryFlipped: once this url has already failed once, retry it via // whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl() const flip = config.proxy.retryFlipped && attempt > 0; const res = await limiterFor(url).schedule(() => bhttp.get(url, { headers: context.headers, agent: agentFor(url, { flip }), // bhttp has no timeout at all by default - a server (or, more likely, a proxy // tunnel) that connects fine but then just stops responding would otherwise hang // forever. This isn't just a wall-clock ceiling on the *promise* (an external // Promise.race achieves that, but leaves the real connection dangling) - bhttp // calls req.abort() itself when this fires, actually releasing the socket. Without // that, every stalled attempt leaks one connection onto the shared agent for that // host; enough of those piling up (e.g. after several retries against a host that's // currently black-holing everything, as seen with i.redd.it) exhausts it, and every // *future* request to that host silently queues forever waiting for a socket that's // never coming back - a freeze with no further log lines, not just a slow retry. responseTimeout: config.fetch.timeout, })); if (res.statusCode !== 200) { throw new Error(`Response not OK for ${url} (${res.statusCode})`); } if (!Buffer.isBuffer(res.body)) { throw new Error(`Unexpected response for ${url} (${res.statusCode}): ${res.body}`); } logger.debug(`Fetched '${context.host?.url || url}' (${context.post?.permalink || 'no post'})`); const hash = blake2.createHash('blake2b', { digestLength: 24 }); hash.update(res.body); const contentHash = hash.digest('hex'); const phashResult = context.item?.type?.includes('image/') ? await phash(res.body) : null; return { buffer: res.body, hash: contentHash, phash: phashResult, }; } catch (error) { return retry(error); } } module.exports = fetchItem;