75 lines
3.0 KiB
JavaScript
75 lines
3.0 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const bhttp = require('bhttp');
|
|
const blake2 = require('blake2');
|
|
const phash = require('sharp-phash');
|
|
// const phashDistance = require('sharp-phash/distance');
|
|
|
|
const { agentFor } = require('../proxy');
|
|
const logger = require('../logger')(__filename);
|
|
const limiterFor = require('../limiter');
|
|
|
|
async function fetchItem(url, attempt, context) {
|
|
async function retry(error) {
|
|
logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${context.post ? context.post.permalink : 'no post'})`);
|
|
|
|
if (attempt < config.fetch.retries) {
|
|
return fetchItem(url, attempt + 1, context);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
// throw new Error('Failed it!');
|
|
// config.proxy.retryFlipped: once this url has already failed once, retry it via
|
|
// whichever of proxied/direct is NOT what it'd normally get - see resolveProxyUrl()
|
|
const flip = config.proxy.retryFlipped && attempt > 0;
|
|
|
|
const res = await limiterFor(url).schedule(() => bhttp.get(url, {
|
|
headers: context.headers,
|
|
agent: agentFor(url, { flip }),
|
|
// bhttp has no timeout at all by default - a server (or, more likely, a proxy
|
|
// tunnel) that connects fine but then just stops responding would otherwise hang
|
|
// forever. This isn't just a wall-clock ceiling on the *promise* (an external
|
|
// Promise.race achieves that, but leaves the real connection dangling) - bhttp
|
|
// calls req.abort() itself when this fires, actually releasing the socket. Without
|
|
// that, every stalled attempt leaks one connection onto the shared agent for that
|
|
// host; enough of those piling up (e.g. after several retries against a host that's
|
|
// currently black-holing everything, as seen with i.redd.it) exhausts it, and every
|
|
// *future* request to that host silently queues forever waiting for a socket that's
|
|
// never coming back - a freeze with no further log lines, not just a slow retry.
|
|
responseTimeout: config.fetch.timeout,
|
|
}));
|
|
|
|
if (res.statusCode !== 200) {
|
|
throw new Error(`Response not OK for ${url} (${res.statusCode})`);
|
|
}
|
|
|
|
if (!Buffer.isBuffer(res.body)) {
|
|
throw new Error(`Unexpected response for ${url} (${res.statusCode}): ${res.body}`);
|
|
}
|
|
|
|
logger.debug(`Fetched '${context.host?.url || url}' (${context.post?.permalink || 'no post'})`);
|
|
|
|
const hash = blake2.createHash('blake2b', { digestLength: 24 });
|
|
hash.update(res.body);
|
|
const contentHash = hash.digest('hex');
|
|
|
|
const phashResult = context.item?.type?.includes('image/')
|
|
? await phash(res.body)
|
|
: null;
|
|
|
|
return {
|
|
buffer: res.body,
|
|
hash: contentHash,
|
|
phash: phashResult,
|
|
};
|
|
} catch (error) {
|
|
return retry(error);
|
|
}
|
|
}
|
|
|
|
module.exports = fetchItem;
|