'use strict'; const config = require('config'); const Promise = require('bluebird'); const path = require('path'); const fs = require('fs-extra'); const bhttp = require('bhttp'); const mime = require('mime'); const sharp = require('sharp'); const blake2 = require('blake2'); const knex = require('./knex'); const pluckPhotos = require('./utils/pluck-photos'); function getHash(buffer) { const hash = blake2.createHash('blake2b', { digestLength: 24 }); hash.update(buffer); return hash.digest('hex'); } async function getThumbnail(buffer) { return sharp(buffer) .resize({ height: config.media.thumbnailSize, withoutEnlargement: true, }) .toBuffer(); } async function createReleaseMediaDirectory(release, releaseId) { if (release.poster || (release.photos && release.photos.length) || release.trailer) { await fs.mkdir( path.join(config.media.path, 'releases', release.site.network.slug, release.site.slug, releaseId.toString()), { recursive: true }, ); } } async function createActorMediaDirectory(profile, actor) { if (profile.avatars && profile.avatars.length) { await fs.mkdir( path.join(config.media.path, 'actors', actor.slug), { recursive: true }, ); } } function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId, setAvatar = false) { return files.map((file, index) => ({ path: file.filepath, thumbnail: file.thumbpath, mime: file.mimetype, hash: file.hash, source: file.source, index, domain, target_id: targetId, role: setAvatar && index === 0 ? 'avatar' : role, })); } // before fetching async function filterSourceDuplicates(photos, domains = ['releases'], roles = ['photo'], identifier) { const photoSourceEntries = await knex('media') .whereIn('source', photos) .whereIn('domain', [].concat(domains)) .whereIn('role', [].concat(roles)); // accept string argument const photoSources = new Set(photoSourceEntries.map(photo => photo.source)); const newPhotos = photos.filter(source => !photoSources.has(source)); if (photoSourceEntries.length > 0) { console.log(`Ignoring ${photoSourceEntries.length} ${roles} items already present by source for ${identifier}`); } return newPhotos; } // after fetching async function filterHashDuplicates(files, domains = ['releases'], roles = ['photo'], identifier) { const photoHashEntries = await knex('media') .whereIn('hash', files.map(file => file.hash)) .whereIn('domain', [].concat(domains)) .whereIn('role', [].concat(roles)); // accept string argument const photoHashes = new Set(photoHashEntries.map(entry => entry.hash)); if (photoHashEntries.length > 0) { console.log(`Ignoring ${photoHashEntries.length} ${roles} items already present by hash for ${identifier}`); } return files.filter(file => file && !photoHashes.has(file.hash)); } async function fetchPhoto(photoUrl, index, identifier) { try { const { pathname } = new URL(photoUrl); const mimetype = mime.getType(pathname); const res = await bhttp.get(photoUrl); if (res.statusCode === 200) { const extension = mime.getExtension(mimetype); const hash = getHash(res.body); return { photo: res.body, mimetype, extension, hash, source: photoUrl, }; } throw new Error(`Response ${res.statusCode} not OK`); } catch (error) { console.warn(`Failed to store photo ${index + 1} (${photoUrl}) for ${identifier}: ${error}`); return null; } } async function savePhotos(files, release, releaseId, actorSlug, isPoster = false) { return Promise.map(files, async (file, index) => { const timestamp = new Date().getTime(); const thumbnail = await getThumbnail(file.photo); const filepath = actorSlug ? path.join('actors', actorSlug, `${timestamp + index}.${file.extension}`) : path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}.${file.extension}`); const thumbpath = actorSlug ? path.join('actors', actorSlug, `${timestamp + index}_thumb.${file.extension}`) : path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}_thumb.${file.extension}`); await Promise.all([ fs.writeFile(path.join(config.media.path, filepath), file.photo), fs.writeFile(path.join(config.media.path, thumbpath), thumbnail), ]); return { ...file, thumbnail, filepath, thumbpath, }; }); } async function storePoster(release, releaseId) { if (!release.poster) { console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`); return; } const [newPoster] = await filterSourceDuplicates([release.poster], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`); if (!newPoster) return; console.log(`Fetching poster for (${release.site.name}, ${releaseId}) "${release.title}"`); const metaFile = await fetchPhoto(release.poster, null, `(${release.site.name}, ${releaseId}) "${release.title}"`); const [uniquePoster] = await filterHashDuplicates([metaFile], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`); if (!uniquePoster) return; const savedPosters = await savePhotos([uniquePoster], release, releaseId, null, true); await knex('media').insert(curatePhotoEntries(savedPosters, 'releases', 'poster', releaseId)); } async function storePhotos(release, releaseId) { if (!release.photos || release.photos.length === 0) { console.warn(`No photos available for (${release.site.name}, ${releaseId}) "${release.title}"`); return; } const pluckedPhotos = pluckPhotos(release.photos, release); console.log(release.photos, pluckedPhotos); const newPhotos = await filterSourceDuplicates(pluckedPhotos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`); if (newPhotos.length === 0) return; console.log(`Fetching ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`); const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), { concurrency: 10, }).filter(photo => photo); const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`); const savedPhotos = await savePhotos(uniquePhotos, release, releaseId); await knex('media').insert(curatePhotoEntries(savedPhotos, 'releases', 'photo', releaseId)); console.log(`Stored ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`); } async function storeTrailer(release, releaseId) { // support scrapers supplying multiple qualities const trailer = Array.isArray(release.trailer) ? release.trailer[0] : release.trailer; if (!trailer || !trailer.src) { console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`); return; } console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`); const { pathname } = new URL(trailer.src); const mimetype = trailer.type || mime.getType(pathname); const res = await bhttp.get(trailer.src); const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`); await Promise.all([ fs.writeFile(path.join(config.media.path, filepath), res.body), knex('media').insert({ path: filepath, mime: mimetype, source: trailer.src, domain: 'releases', target_id: releaseId, role: 'trailer', quality: trailer.quality || null, }), ]); } async function storeAvatars(profile, actor) { if (!profile.avatars || profile.avatars.length === 0) { console.warn(`No avatars available for '${profile.name}'`); return; } const newPhotos = await filterSourceDuplicates(profile.avatars, 'actors', ['avatar', 'photo'], actor.name); if (newPhotos.length === 0) return; console.log(`Fetching ${newPhotos.length} avatars for '${actor.name}'`); const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), { concurrency: 10, }).filter(photo => photo); const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name); const [savedPhotos, avatarEntry] = await Promise.all([ savePhotos(uniquePhotos, null, null, actor.slug), knex('media').where({ target_id: actor.id, domain: 'actors', role: 'avatar', }).first(), ]); // if no avatar entry is present, curatePhotoEntries will store the first photo as avatar await knex('media').insert(curatePhotoEntries(savedPhotos, 'actors', 'photo', actor.id, !avatarEntry)); } module.exports = { createActorMediaDirectory, createReleaseMediaDirectory, storeAvatars, storePoster, storePhotos, storeTrailer, };