traxxx/src/media.js

'use strict';

const config = require('config');
const Promise = require('bluebird');
const bhttp = require('bhttp');
const mime = require('mime');
const sharp = require('sharp');
const blake2 = require('blake2');

const logger = require('./logger');
const knex = require('./knex');

function getHash(buffer) {
    const hash = blake2.createHash('blake2b', { digestLength: 24 });
    hash.update(buffer);

    return hash.digest('hex');
}

function pluckItems(items, specifiedLimit) {
    const limit = specifiedLimit || config.media.limit;

    if (items.length <= limit) return items;

    const plucked = [1]
        .concat(
            Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (items.length / (limit - 1)))),
        );

    return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
}

async function getEntropy(buffer) {
    try {
        const { entropy } = await sharp(buffer).stats();

        return entropy;
    } catch (error) {
        logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`);

        return 7.5;
    }
}

async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
    try {
        if (Array.isArray(source)) {
            // fallbacks provided
            return source.reduce((outcome, sourceX) => outcome.catch(async () => {
                const item = await fetchItem(sourceX, index, existingItemsBySource);

                if (item) {
                    return item;
                }

                throw new Error(`Item not available: ${source}`);
            }), Promise.reject(new Error()));
        }

        if (existingItemsBySource[source]) {
            return existingItemsBySource[source];
        }

        const res = await bhttp.get(source);

        if (res.statusCode === 200) {
            const { pathname } = new URL(source);
            const mimetype = mime.getType(pathname);
            const extension = mime.getExtension(mimetype);
            const hash = getHash(res.body);
            const entropy = await getEntropy(res.body);

            return {
                file: res.body,
                mimetype,
                extension,
                hash,
                entropy,
                source,
            };
        }

        throw new Error(`Response ${res.statusCode} not OK`);
    } catch (error) {
        if (attempt <= 3) {
            return fetchItem(source, index, existingItemsBySource, attempt + 1);
        }

        throw new Error(`Failed to fetch media from ${source}: ${error}`);
    }
}

async function fetchItems(itemSources, existingItemsBySource) {
    return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource));
}

async function storeReleaseMedia(releases, {
    type = 'poster',
} = {}) {
    const pluckedSources = releases.map(release => pluckItems(release[type]));
    const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat());
    const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {});

    const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource);
    const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash));
    const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {});

    const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]);

    console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems);
}

module.exports = {
    storeReleaseMedia,
};