2019-11-11 02:20:00 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
|
|
|
const Promise = require('bluebird');
|
|
|
|
const path = require('path');
|
|
|
|
const fs = require('fs-extra');
|
|
|
|
const bhttp = require('bhttp');
|
|
|
|
const mime = require('mime');
|
|
|
|
const sharp = require('sharp');
|
|
|
|
const blake2 = require('blake2');
|
|
|
|
|
|
|
|
const knex = require('./knex');
|
|
|
|
|
|
|
|
function getHash(buffer) {
|
|
|
|
const hash = blake2.createHash('blake2b', { digestLength: 24 });
|
|
|
|
|
|
|
|
hash.update(buffer);
|
|
|
|
|
|
|
|
return hash.digest('hex');
|
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
function pluckPhotos(photos, specifiedLimit) {
|
2019-12-12 02:12:05 +00:00
|
|
|
const limit = specifiedLimit || config.media.limit;
|
|
|
|
|
|
|
|
if (photos.length <= limit) {
|
|
|
|
return photos;
|
|
|
|
}
|
|
|
|
|
|
|
|
const plucked = [1]
|
|
|
|
.concat(
|
2019-12-12 03:04:35 +00:00
|
|
|
Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (photos.length / (limit - 1)))),
|
2019-12-12 02:12:05 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
|
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function createThumbnail(buffer) {
|
2019-11-12 00:22:20 +00:00
|
|
|
return sharp(buffer)
|
|
|
|
.resize({
|
|
|
|
height: config.media.thumbnailSize,
|
|
|
|
withoutEnlargement: true,
|
|
|
|
})
|
|
|
|
.toBuffer();
|
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function createMediaDirectory(domain, subpath) {
|
|
|
|
const filepath = path.join(config.media.path, domain, subpath);
|
2019-11-20 03:53:36 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
await fs.mkdir(filepath, { recursive: true });
|
|
|
|
return filepath;
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId) {
|
2019-12-07 02:59:55 +00:00
|
|
|
return files.map((file, index) => ({
|
|
|
|
path: file.filepath,
|
|
|
|
thumbnail: file.thumbpath,
|
|
|
|
mime: file.mimetype,
|
|
|
|
hash: file.hash,
|
|
|
|
source: file.source,
|
|
|
|
index,
|
|
|
|
domain,
|
|
|
|
target_id: targetId,
|
2019-12-13 02:28:52 +00:00
|
|
|
role: file.role || role,
|
2019-12-07 02:59:55 +00:00
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
|
|
|
// before fetching
|
|
|
|
async function filterSourceDuplicates(photos, domains = ['releases'], roles = ['photo'], identifier) {
|
|
|
|
const photoSourceEntries = await knex('media')
|
2019-12-12 03:04:35 +00:00
|
|
|
.whereIn('source', photos.flat())
|
2019-12-13 02:28:52 +00:00
|
|
|
.whereIn('domain', domains)
|
|
|
|
.whereIn('role', roles); // accept string argument
|
2019-12-07 02:59:55 +00:00
|
|
|
|
|
|
|
const photoSources = new Set(photoSourceEntries.map(photo => photo.source));
|
2019-12-12 03:04:35 +00:00
|
|
|
const newPhotos = photos.filter(source => (Array.isArray(source) // fallbacks provided?
|
|
|
|
? !source.some(sourceX => photoSources.has(sourceX)) // ensure none of the sources match
|
|
|
|
: !photoSources.has(source)));
|
2019-12-07 02:59:55 +00:00
|
|
|
|
|
|
|
if (photoSourceEntries.length > 0) {
|
|
|
|
console.log(`Ignoring ${photoSourceEntries.length} ${roles} items already present by source for ${identifier}`);
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return newPhotos;
|
|
|
|
}
|
|
|
|
|
|
|
|
// after fetching
|
|
|
|
async function filterHashDuplicates(files, domains = ['releases'], roles = ['photo'], identifier) {
|
|
|
|
const photoHashEntries = await knex('media')
|
|
|
|
.whereIn('hash', files.map(file => file.hash))
|
|
|
|
.whereIn('domain', [].concat(domains))
|
|
|
|
.whereIn('role', [].concat(roles)); // accept string argument
|
|
|
|
|
|
|
|
const photoHashes = new Set(photoHashEntries.map(entry => entry.hash));
|
|
|
|
|
|
|
|
if (photoHashEntries.length > 0) {
|
|
|
|
console.log(`Ignoring ${photoHashEntries.length} ${roles} items already present by hash for ${identifier}`);
|
|
|
|
}
|
|
|
|
|
|
|
|
return files.filter(file => file && !photoHashes.has(file.hash));
|
|
|
|
}
|
|
|
|
|
2019-12-12 02:12:05 +00:00
|
|
|
async function fetchPhoto(photoUrl, index, identifier, attempt = 1) {
|
|
|
|
if (Array.isArray(photoUrl)) {
|
2019-12-12 04:18:43 +00:00
|
|
|
return photoUrl.reduce(async (outcome, url) => outcome.catch(async () => {
|
|
|
|
const photo = await fetchPhoto(url, index, identifier);
|
|
|
|
|
|
|
|
if (photo) {
|
|
|
|
return photo;
|
|
|
|
}
|
|
|
|
|
|
|
|
throw new Error('Photo not available');
|
|
|
|
}), Promise.reject(new Error()));
|
2019-12-12 02:12:05 +00:00
|
|
|
}
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
try {
|
2019-12-09 04:00:49 +00:00
|
|
|
const { pathname } = new URL(photoUrl);
|
|
|
|
const mimetype = mime.getType(pathname);
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const res = await bhttp.get(photoUrl);
|
|
|
|
|
|
|
|
if (res.statusCode === 200) {
|
|
|
|
const extension = mime.getExtension(mimetype);
|
|
|
|
const hash = getHash(res.body);
|
|
|
|
|
|
|
|
return {
|
|
|
|
photo: res.body,
|
|
|
|
mimetype,
|
|
|
|
extension,
|
|
|
|
hash,
|
|
|
|
source: photoUrl,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
throw new Error(`Response ${res.statusCode} not OK`);
|
|
|
|
} catch (error) {
|
2019-12-12 03:04:35 +00:00
|
|
|
console.warn(`Failed attempt ${attempt}/3 to fetch photo ${index + 1} for ${identifier} (${photoUrl}): ${error}`);
|
2019-12-12 02:12:05 +00:00
|
|
|
|
|
|
|
if (attempt < 3) {
|
|
|
|
await Promise.delay(1000);
|
|
|
|
return fetchPhoto(photoUrl, index, identifier, attempt + 1);
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function savePhotos(files, {
|
|
|
|
domain = 'releases',
|
|
|
|
subpath,
|
|
|
|
role = 'photo',
|
|
|
|
naming = 'index',
|
|
|
|
}) {
|
2019-12-07 02:59:55 +00:00
|
|
|
return Promise.map(files, async (file, index) => {
|
|
|
|
const timestamp = new Date().getTime();
|
2019-12-13 02:28:52 +00:00
|
|
|
const thumbnail = await createThumbnail(file.photo);
|
2019-11-20 03:53:36 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const filename = naming === 'index'
|
2019-12-13 15:59:04 +00:00
|
|
|
? `${file.role || role}-${index + 1}`
|
2019-12-13 02:28:52 +00:00
|
|
|
: `${timestamp + index}`;
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const filepath = path.join(domain, subpath, `${filename}.${file.extension}`);
|
|
|
|
const thumbpath = path.join(domain, subpath, `${filename}_thumb.${file.extension}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
|
|
|
await Promise.all([
|
2019-12-07 02:59:55 +00:00
|
|
|
fs.writeFile(path.join(config.media.path, filepath), file.photo),
|
2019-11-11 02:20:00 +00:00
|
|
|
fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
|
|
|
|
]);
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return {
|
|
|
|
...file,
|
|
|
|
thumbnail,
|
|
|
|
filepath,
|
|
|
|
thumbpath,
|
|
|
|
};
|
|
|
|
});
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function storePhotos(photos, {
|
|
|
|
domain = 'releases',
|
|
|
|
role = 'photo',
|
|
|
|
naming = 'index',
|
|
|
|
targetId,
|
|
|
|
subpath,
|
|
|
|
primaryRole, // role to assign to first photo if not already in database, used mainly for avatars
|
|
|
|
}, identifier) {
|
|
|
|
if (!photos || photos.length === 0) {
|
|
|
|
console.warn(`No ${role}s available for ${identifier}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
return;
|
|
|
|
}
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const pluckedPhotos = pluckPhotos(photos);
|
|
|
|
const roles = primaryRole ? [role, primaryRole] : [role];
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const newPhotos = await filterSourceDuplicates(pluckedPhotos, [domain], roles, identifier);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
if (newPhotos.length === 0) return;
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
console.log(`Fetching ${newPhotos.length} ${role}s for ${identifier}`);
|
2019-12-07 02:59:55 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, identifier), {
|
2019-12-04 20:58:08 +00:00
|
|
|
concurrency: 10,
|
2019-12-09 23:30:55 +00:00
|
|
|
}).filter(photo => photo);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const [uniquePhotos, primaryPhoto] = await Promise.all([
|
|
|
|
filterHashDuplicates(metaFiles, [domain], roles, identifier),
|
|
|
|
primaryRole
|
|
|
|
? await knex('media')
|
|
|
|
.where('domain', domain)
|
|
|
|
.where('target_id', targetId)
|
|
|
|
.where('role', primaryRole)
|
|
|
|
.first()
|
|
|
|
: null,
|
|
|
|
]);
|
|
|
|
|
|
|
|
if (primaryRole && !primaryPhoto) {
|
2019-12-13 15:59:04 +00:00
|
|
|
console.log(`Setting first photo as ${primaryRole} for ${identifier}`);
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
uniquePhotos[0].role = primaryRole;
|
|
|
|
}
|
2019-12-12 04:18:43 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const savedPhotos = await savePhotos(uniquePhotos, {
|
|
|
|
domain,
|
|
|
|
role,
|
|
|
|
targetId,
|
|
|
|
subpath,
|
|
|
|
naming,
|
|
|
|
});
|
2019-12-07 02:59:55 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
const curatedPhotoEntries = curatePhotoEntries(savedPhotos, domain, role, targetId);
|
2019-12-12 03:04:35 +00:00
|
|
|
|
|
|
|
await knex('media').insert(curatedPhotoEntries);
|
2019-12-07 02:59:55 +00:00
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
console.log(`Stored ${newPhotos.length} ${role}s for ${identifier}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function storeTrailer(trailers, {
|
|
|
|
domain = 'releases',
|
|
|
|
role = 'trailer',
|
|
|
|
targetId,
|
|
|
|
subpath,
|
|
|
|
}, identifier) {
|
2019-12-07 03:17:14 +00:00
|
|
|
// support scrapers supplying multiple qualities
|
2019-12-13 02:28:52 +00:00
|
|
|
const trailer = Array.isArray(trailers)
|
|
|
|
? trailers.find(trailerX => [1080, 720].includes(trailerX.quality)) || trailers[0]
|
|
|
|
: trailers;
|
2019-12-07 03:17:14 +00:00
|
|
|
|
|
|
|
if (!trailer || !trailer.src) {
|
2019-12-13 02:28:52 +00:00
|
|
|
console.warn(`No trailer available for ${identifier}`);
|
2019-11-16 02:33:36 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
console.log(`Storing trailer for ${identifier}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 03:17:14 +00:00
|
|
|
const { pathname } = new URL(trailer.src);
|
|
|
|
const mimetype = trailer.type || mime.getType(pathname);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 03:17:14 +00:00
|
|
|
const res = await bhttp.get(trailer.src);
|
2019-12-13 02:28:52 +00:00
|
|
|
const filepath = path.join('releases', subpath, `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
|
|
|
await Promise.all([
|
|
|
|
fs.writeFile(path.join(config.media.path, filepath), res.body),
|
|
|
|
knex('media').insert({
|
|
|
|
path: filepath,
|
|
|
|
mime: mimetype,
|
2019-12-07 03:17:14 +00:00
|
|
|
source: trailer.src,
|
2019-12-13 02:28:52 +00:00
|
|
|
domain,
|
|
|
|
target_id: targetId,
|
|
|
|
role,
|
2019-12-07 03:17:14 +00:00
|
|
|
quality: trailer.quality || null,
|
2019-11-11 02:20:00 +00:00
|
|
|
}),
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
|
2019-12-13 02:28:52 +00:00
|
|
|
async function findAvatar(actorId, domain = 'actors') {
|
|
|
|
return knex('media')
|
|
|
|
.where('domain', domain)
|
|
|
|
.where('target_id', actorId)
|
|
|
|
.where('role', 'avatar');
|
2019-11-20 03:53:36 +00:00
|
|
|
}
|
|
|
|
|
2019-11-11 02:20:00 +00:00
|
|
|
module.exports = {
|
2019-12-13 02:28:52 +00:00
|
|
|
createMediaDirectory,
|
|
|
|
findAvatar,
|
2019-11-11 02:20:00 +00:00
|
|
|
storePhotos,
|
|
|
|
storeTrailer,
|
|
|
|
};
|