'use strict'; const config = require('config'); const Promise = require('bluebird'); const bhttp = require('bhttp'); const mime = require('mime'); const sharp = require('sharp'); const blake2 = require('blake2'); const logger = require('./logger'); const knex = require('./knex'); function getHash(buffer) { const hash = blake2.createHash('blake2b', { digestLength: 24 }); hash.update(buffer); return hash.digest('hex'); } function pluckItems(items, specifiedLimit) { const limit = specifiedLimit || config.media.limit; if (items.length <= limit) return items; const plucked = [1] .concat( Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (items.length / (limit - 1)))), ); return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close } async function getEntropy(buffer) { try { const { entropy } = await sharp(buffer).stats(); return entropy; } catch (error) { logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`); return 7.5; } } async function fetchItem(source, index, existingItemsBySource, attempt = 1) { try { if (Array.isArray(source)) { // fallbacks provided return source.reduce((outcome, sourceX) => outcome.catch(async () => { const item = await fetchItem(sourceX, index, existingItemsBySource); if (item) { return item; } throw new Error(`Item not available: ${source}`); }), Promise.reject(new Error())); } if (existingItemsBySource[source]) { return existingItemsBySource[source]; } const res = await bhttp.get(source); if (res.statusCode === 200) { const { pathname } = new URL(source); const mimetype = mime.getType(pathname); const extension = mime.getExtension(mimetype); const hash = getHash(res.body); const entropy = await getEntropy(res.body); return { file: res.body, mimetype, extension, hash, entropy, source, }; } throw new Error(`Response ${res.statusCode} not OK`); } catch (error) { if (attempt <= 3) { return fetchItem(source, index, existingItemsBySource, attempt + 1); } throw new Error(`Failed to fetch media from ${source}: ${error}`); } } async function fetchItems(itemSources, existingItemsBySource) { return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource)); } async function storeReleaseMedia(releases, { type = 'poster', } = {}) { const pluckedSources = releases.map(release => pluckItems(release[type])); const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat()); const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {}); const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource); const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash)); const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {}); const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]); console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems); } module.exports = { storeReleaseMedia, };