traxxx/src/media.js

'use strict';

const config = require('config');
const Promise = require('bluebird');
const path = require('path');
const fs = require('fs-extra');
const bhttp = require('bhttp');
const mime = require('mime');
const sharp = require('sharp');
const blake2 = require('blake2');

const knex = require('./knex');
const pluckPhotos = require('./utils/pluck-photos');

function getHash(buffer) {
    const hash = blake2.createHash('blake2b', { digestLength: 24 });

    hash.update(buffer);

    return hash.digest('hex');
}

async function getThumbnail(buffer) {
    return sharp(buffer)
        .resize({
            height: config.media.thumbnailSize,
            withoutEnlargement: true,
        })
        .toBuffer();
}

async function createReleaseMediaDirectory(release, releaseId) {
    if (release.poster || (release.photos && release.photos.length) || release.trailer) {
        await fs.mkdir(
            path.join(config.media.path, 'releases', release.site.network.slug, release.site.slug, releaseId.toString()),
            { recursive: true },
        );
    }
}

async function createActorMediaDirectory(profile, actor) {
    if (profile.avatars && profile.avatars.length) {
        await fs.mkdir(
            path.join(config.media.path, 'actors', actor.slug),
            { recursive: true },
        );
    }
}

function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId, setAvatar = false) {
    return files.map((file, index) => ({
        path: file.filepath,
        thumbnail: file.thumbpath,
        mime: file.mimetype,
        hash: file.hash,
        source: file.source,
        index,
        domain,
        target_id: targetId,
        role: setAvatar && index === 0 ? 'avatar' : role,
    }));
}

// before fetching
async function filterSourceDuplicates(photos, domains = ['releases'], roles = ['photo'], identifier) {
    const photoSourceEntries = await knex('media')
        .whereIn('source', photos)
        .whereIn('domain', [].concat(domains))
        .whereIn('role', [].concat(roles)); // accept string argument

    const photoSources = new Set(photoSourceEntries.map(photo => photo.source));
    const newPhotos = photos.filter(source => !photoSources.has(source));

    if (photoSourceEntries.length > 0) {
        console.log(`Ignoring ${photoSourceEntries.length} ${roles} items already present by source for ${identifier}`);
    }

    return newPhotos;
}

// after fetching
async function filterHashDuplicates(files, domains = ['releases'], roles = ['photo'], identifier) {
    const photoHashEntries = await knex('media')
        .whereIn('hash', files.map(file => file.hash))
        .whereIn('domain', [].concat(domains))
        .whereIn('role', [].concat(roles)); // accept string argument

    const photoHashes = new Set(photoHashEntries.map(entry => entry.hash));

    if (photoHashEntries.length > 0) {
        console.log(`Ignoring ${photoHashEntries.length} ${roles} items already present by hash for ${identifier}`);
    }

    return files.filter(file => file && !photoHashes.has(file.hash));
}

async function fetchPhoto(photoUrl, index, identifier) {
    try {
        const { pathname } = new URL(photoUrl);
        const mimetype = mime.getType(pathname);

        const res = await bhttp.get(photoUrl);

        if (res.statusCode === 200) {
            const extension = mime.getExtension(mimetype);
            const hash = getHash(res.body);

            return {
                photo: res.body,
                mimetype,
                extension,
                hash,
                source: photoUrl,
            };
        }

        throw new Error(`Response ${res.statusCode} not OK`);
    } catch (error) {
        console.warn(`Failed to store photo ${index + 1} (${photoUrl}) for ${identifier}: ${error}`);

        return null;
    }
}

async function savePhotos(files, release, releaseId, actorSlug, isPoster = false) {
    return Promise.map(files, async (file, index) => {
        const timestamp = new Date().getTime();
        const thumbnail = await getThumbnail(file.photo);

        const filepath = actorSlug
            ? path.join('actors', actorSlug, `${timestamp + index}.${file.extension}`)
            : path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}.${file.extension}`);

        const thumbpath = actorSlug
            ? path.join('actors', actorSlug, `${timestamp + index}_thumb.${file.extension}`)
            : path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}_thumb.${file.extension}`);

        await Promise.all([
            fs.writeFile(path.join(config.media.path, filepath), file.photo),
            fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
        ]);

        return {
            ...file,
            thumbnail,
            filepath,
            thumbpath,
        };
    });
}

async function storePoster(release, releaseId) {
    if (!release.poster) {
        console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
        return;
    }
    const [newPoster] = await filterSourceDuplicates([release.poster], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);

    if (!newPoster) return;

    console.log(`Fetching poster for (${release.site.name}, ${releaseId}) "${release.title}"`);

    const metaFile = await fetchPhoto(release.poster, null, `(${release.site.name}, ${releaseId}) "${release.title}"`);
    const [uniquePoster] = await filterHashDuplicates([metaFile], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);

    if (!uniquePoster) return;

    const savedPosters = await savePhotos([uniquePoster], release, releaseId, null, true);

    await knex('media').insert(curatePhotoEntries(savedPosters, 'releases', 'poster', releaseId));
}


async function storePhotos(release, releaseId) {
    if (!release.photos || release.photos.length === 0) {
        console.warn(`No photos available for (${release.site.name}, ${releaseId}) "${release.title}"`);
        return;
    }

    const pluckedPhotos = pluckPhotos(release.photos, release);

    console.log(release.photos, pluckedPhotos);

    const newPhotos = await filterSourceDuplicates(pluckedPhotos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);

    if (newPhotos.length === 0) return;

    console.log(`Fetching ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);

    const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), {
        concurrency: 10,
    }).filter(photo => photo);

    const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
    const savedPhotos = await savePhotos(uniquePhotos, release, releaseId);

    await knex('media').insert(curatePhotoEntries(savedPhotos, 'releases', 'photo', releaseId));

    console.log(`Stored ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
}

async function storeTrailer(release, releaseId) {
    // support scrapers supplying multiple qualities
    const trailer = Array.isArray(release.trailer) ? release.trailer[0] : release.trailer;

    if (!trailer || !trailer.src) {
        console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
        return;
    }

    console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`);

    const { pathname } = new URL(trailer.src);
    const mimetype = trailer.type || mime.getType(pathname);

    const res = await bhttp.get(trailer.src);
    const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);

    await Promise.all([
        fs.writeFile(path.join(config.media.path, filepath), res.body),
        knex('media').insert({
            path: filepath,
            mime: mimetype,
            source: trailer.src,
            domain: 'releases',
            target_id: releaseId,
            role: 'trailer',
            quality: trailer.quality || null,
        }),
    ]);
}

async function storeAvatars(profile, actor) {
    if (!profile.avatars || profile.avatars.length === 0) {
        console.warn(`No avatars available for '${profile.name}'`);
        return;
    }

    const newPhotos = await filterSourceDuplicates(profile.avatars, 'actors', ['avatar', 'photo'], actor.name);

    if (newPhotos.length === 0) return;

    console.log(`Fetching ${newPhotos.length} avatars for '${actor.name}'`);

    const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), {
        concurrency: 10,
    }).filter(photo => photo);

    const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name);
    const [savedPhotos, avatarEntry] = await Promise.all([
        savePhotos(uniquePhotos, null, null, actor.slug),
        knex('media').where({
            target_id: actor.id,
            domain: 'actors',
            role: 'avatar',
        }).first(),
    ]);

    // if no avatar entry is present, curatePhotoEntries will store the first photo as avatar
    await knex('media').insert(curatePhotoEntries(savedPhotos, 'actors', 'photo', actor.id, !avatarEntry));
}

module.exports = {
    createActorMediaDirectory,
    createReleaseMediaDirectory,
    storeAvatars,
    storePoster,
    storePhotos,
    storeTrailer,
};