'use strict'; const config = require('config'); const bhttp = require('bhttp'); const blake2 = require('blake2'); const logger = require('../logger')(__filename); const limiter = require('../limiter').items; async function fetchItem(url, attempt, post, host) { async function retry(error) { logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`); if (attempt < config.fetch.retries) { return fetchItem(url, attempt + 1, post); } return null; } try { const res = await limiter.schedule(async () => bhttp.get(url)); if (!res.statusCode === 200) { throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`); } if (!Buffer.isBuffer(res.body)) { throw new Error(`Unexpected response for '${url}' (${res.status}): ${res.body}`); } logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`); const hash = blake2.createHash('blake2b', { digestLength: 24 }); hash.update(res.body); const contentHash = hash.digest('hex'); return Object.assign(res.body, { hash: contentHash }); } catch (error) { return retry(error); } } module.exports = fetchItem;