45 lines
1.3 KiB
JavaScript
45 lines
1.3 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const bhttp = require('bhttp');
|
|
const blake2 = require('blake2');
|
|
|
|
const logger = require('../logger')(__filename);
|
|
const limiter = require('../limiter').items;
|
|
|
|
async function fetchItem(url, attempt, post, host) {
|
|
async function retry(error) {
|
|
logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`);
|
|
|
|
if (attempt < config.fetch.retries) {
|
|
return fetchItem(url, attempt + 1, post);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const res = await limiter.schedule(async () => bhttp.get(url));
|
|
|
|
if (!res.statusCode === 200) {
|
|
throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`);
|
|
}
|
|
|
|
if (!Buffer.isBuffer(res.body)) {
|
|
throw new Error(`Unexpected response for '${url}' (${res.status}): ${res.body}`);
|
|
}
|
|
|
|
logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`);
|
|
|
|
const hash = blake2.createHash('blake2b', { digestLength: 24 });
|
|
hash.update(res.body);
|
|
const contentHash = hash.digest('hex');
|
|
|
|
return Object.assign(res.body, { hash: contentHash });
|
|
} catch (error) {
|
|
return retry(error);
|
|
}
|
|
}
|
|
|
|
module.exports = fetchItem;
|