forked from DebaucheryLibrarian/traxxx
116 lines
3.6 KiB
JavaScript
116 lines
3.6 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const Promise = require('bluebird');
|
|
const bhttp = require('bhttp');
|
|
const mime = require('mime');
|
|
const sharp = require('sharp');
|
|
const blake2 = require('blake2');
|
|
|
|
const logger = require('./logger');
|
|
const knex = require('./knex');
|
|
|
|
function getHash(buffer) {
|
|
const hash = blake2.createHash('blake2b', { digestLength: 24 });
|
|
hash.update(buffer);
|
|
|
|
return hash.digest('hex');
|
|
}
|
|
|
|
function pluckItems(items, specifiedLimit) {
|
|
const limit = specifiedLimit || config.media.limit;
|
|
|
|
if (items.length <= limit) return items;
|
|
|
|
const plucked = [1]
|
|
.concat(
|
|
Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (items.length / (limit - 1)))),
|
|
);
|
|
|
|
return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
|
|
}
|
|
|
|
async function getEntropy(buffer) {
|
|
try {
|
|
const { entropy } = await sharp(buffer).stats();
|
|
|
|
return entropy;
|
|
} catch (error) {
|
|
logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`);
|
|
|
|
return 7.5;
|
|
}
|
|
}
|
|
|
|
async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
|
|
try {
|
|
if (Array.isArray(source)) {
|
|
// fallbacks provided
|
|
return source.reduce((outcome, sourceX) => outcome.catch(async () => {
|
|
const item = await fetchItem(sourceX, index, existingItemsBySource);
|
|
|
|
if (item) {
|
|
return item;
|
|
}
|
|
|
|
throw new Error(`Item not available: ${source}`);
|
|
}), Promise.reject(new Error()));
|
|
}
|
|
|
|
if (existingItemsBySource[source]) {
|
|
return existingItemsBySource[source];
|
|
}
|
|
|
|
const res = await bhttp.get(source);
|
|
|
|
if (res.statusCode === 200) {
|
|
const { pathname } = new URL(source);
|
|
const mimetype = mime.getType(pathname);
|
|
const extension = mime.getExtension(mimetype);
|
|
const hash = getHash(res.body);
|
|
const entropy = await getEntropy(res.body);
|
|
|
|
return {
|
|
file: res.body,
|
|
mimetype,
|
|
extension,
|
|
hash,
|
|
entropy,
|
|
source,
|
|
};
|
|
}
|
|
|
|
throw new Error(`Response ${res.statusCode} not OK`);
|
|
} catch (error) {
|
|
if (attempt <= 3) {
|
|
return fetchItem(source, index, existingItemsBySource, attempt + 1);
|
|
}
|
|
|
|
throw new Error(`Failed to fetch media from ${source}: ${error}`);
|
|
}
|
|
}
|
|
|
|
async function fetchItems(itemSources, existingItemsBySource) {
|
|
return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource));
|
|
}
|
|
|
|
async function storeReleaseMedia(releases, {
|
|
type = 'poster',
|
|
} = {}) {
|
|
const pluckedSources = releases.map(release => pluckItems(release[type]));
|
|
const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat());
|
|
const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {});
|
|
|
|
const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource);
|
|
const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash));
|
|
const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {});
|
|
|
|
const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]);
|
|
|
|
console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems);
|
|
}
|
|
|
|
module.exports = {
|
|
storeReleaseMedia,
|
|
};
|