2019-11-11 02:20:00 +00:00
|
|
|
'use strict';
|
|
|
|
|
|
|
|
const config = require('config');
|
|
|
|
const Promise = require('bluebird');
|
|
|
|
const path = require('path');
|
|
|
|
const fs = require('fs-extra');
|
|
|
|
const bhttp = require('bhttp');
|
|
|
|
const mime = require('mime');
|
|
|
|
const sharp = require('sharp');
|
|
|
|
const blake2 = require('blake2');
|
|
|
|
|
|
|
|
const knex = require('./knex');
|
|
|
|
|
|
|
|
function getHash(buffer) {
|
|
|
|
const hash = blake2.createHash('blake2b', { digestLength: 24 });
|
|
|
|
|
|
|
|
hash.update(buffer);
|
|
|
|
|
|
|
|
return hash.digest('hex');
|
|
|
|
}
|
|
|
|
|
2019-11-12 00:22:20 +00:00
|
|
|
async function getThumbnail(buffer) {
|
|
|
|
return sharp(buffer)
|
|
|
|
.resize({
|
|
|
|
height: config.media.thumbnailSize,
|
|
|
|
withoutEnlargement: true,
|
|
|
|
})
|
|
|
|
.toBuffer();
|
|
|
|
}
|
|
|
|
|
2019-11-20 03:53:36 +00:00
|
|
|
async function createReleaseMediaDirectory(release, releaseId) {
|
2019-12-04 20:58:08 +00:00
|
|
|
if (release.poster || (release.photos && release.photos.length) || release.trailer) {
|
2019-11-16 02:33:36 +00:00
|
|
|
await fs.mkdir(
|
2019-11-20 03:53:36 +00:00
|
|
|
path.join(config.media.path, 'releases', release.site.network.slug, release.site.slug, releaseId.toString()),
|
|
|
|
{ recursive: true },
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async function createActorMediaDirectory(profile, actor) {
|
|
|
|
if (profile.avatars && profile.avatars.length) {
|
|
|
|
await fs.mkdir(
|
|
|
|
path.join(config.media.path, 'actors', actor.slug),
|
2019-11-16 02:33:36 +00:00
|
|
|
{ recursive: true },
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId, setAvatar = false) {
|
|
|
|
return files.map((file, index) => ({
|
|
|
|
path: file.filepath,
|
|
|
|
thumbnail: file.thumbpath,
|
|
|
|
mime: file.mimetype,
|
|
|
|
hash: file.hash,
|
|
|
|
source: file.source,
|
|
|
|
index,
|
|
|
|
domain,
|
|
|
|
target_id: targetId,
|
|
|
|
role: setAvatar && index === 0 ? 'avatar' : role,
|
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
|
|
|
// before fetching
|
|
|
|
async function filterSourceDuplicates(photos, domains = ['releases'], roles = ['photo'], identifier) {
|
|
|
|
const photoSourceEntries = await knex('media')
|
|
|
|
.whereIn('source', photos)
|
|
|
|
.whereIn('domain', [].concat(domains))
|
|
|
|
.whereIn('role', [].concat(roles)); // accept string argument
|
|
|
|
|
|
|
|
const photoSources = new Set(photoSourceEntries.map(photo => photo.source));
|
|
|
|
const newPhotos = photos.filter(source => !photoSources.has(source));
|
|
|
|
|
|
|
|
if (photoSourceEntries.length > 0) {
|
|
|
|
console.log(`Ignoring ${photoSourceEntries.length} ${roles} items already present by source for ${identifier}`);
|
2019-11-16 02:33:36 +00:00
|
|
|
}
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return newPhotos;
|
|
|
|
}
|
|
|
|
|
|
|
|
// after fetching
|
|
|
|
async function filterHashDuplicates(files, domains = ['releases'], roles = ['photo'], identifier) {
|
|
|
|
const photoHashEntries = await knex('media')
|
|
|
|
.whereIn('hash', files.map(file => file.hash))
|
|
|
|
.whereIn('domain', [].concat(domains))
|
|
|
|
.whereIn('role', [].concat(roles)); // accept string argument
|
|
|
|
|
|
|
|
const photoHashes = new Set(photoHashEntries.map(entry => entry.hash));
|
|
|
|
|
|
|
|
if (photoHashEntries.length > 0) {
|
|
|
|
console.log(`Ignoring ${photoHashEntries.length} ${roles} items already present by hash for ${identifier}`);
|
|
|
|
}
|
|
|
|
|
|
|
|
return files.filter(file => file && !photoHashes.has(file.hash));
|
|
|
|
}
|
|
|
|
|
|
|
|
async function fetchPhoto(photoUrl, index, identifier) {
|
|
|
|
const { pathname } = new URL(photoUrl);
|
|
|
|
const mimetype = mime.getType(pathname);
|
|
|
|
|
|
|
|
try {
|
|
|
|
const res = await bhttp.get(photoUrl);
|
|
|
|
|
|
|
|
if (res.statusCode === 200) {
|
|
|
|
const extension = mime.getExtension(mimetype);
|
|
|
|
const hash = getHash(res.body);
|
|
|
|
|
|
|
|
return {
|
|
|
|
photo: res.body,
|
|
|
|
mimetype,
|
|
|
|
extension,
|
|
|
|
hash,
|
|
|
|
source: photoUrl,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
throw new Error(`Response ${res.statusCode} not OK`);
|
|
|
|
} catch (error) {
|
|
|
|
console.warn(`Failed to store photo ${index + 1} (${photoUrl}) for ${identifier}: ${error}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
async function savePhotos(files, release, releaseId, actorSlug, isPoster = false) {
|
|
|
|
return Promise.map(files, async (file, index) => {
|
|
|
|
const timestamp = new Date().getTime();
|
|
|
|
const thumbnail = await getThumbnail(file.photo);
|
2019-11-20 03:53:36 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const filepath = actorSlug
|
|
|
|
? path.join('actors', actorSlug, `${timestamp + index}.${file.extension}`)
|
|
|
|
: path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}.${file.extension}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const thumbpath = actorSlug
|
|
|
|
? path.join('actors', actorSlug, `${timestamp + index}_thumb.${file.extension}`)
|
|
|
|
: path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}_thumb.${file.extension}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
|
|
|
await Promise.all([
|
2019-12-07 02:59:55 +00:00
|
|
|
fs.writeFile(path.join(config.media.path, filepath), file.photo),
|
2019-11-11 02:20:00 +00:00
|
|
|
fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
|
|
|
|
]);
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
return {
|
|
|
|
...file,
|
|
|
|
thumbnail,
|
|
|
|
filepath,
|
|
|
|
thumbpath,
|
|
|
|
};
|
|
|
|
});
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
async function storePoster(release, releaseId) {
|
|
|
|
if (!release.poster) {
|
|
|
|
console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
return;
|
|
|
|
}
|
2019-12-07 02:59:55 +00:00
|
|
|
const [newPoster] = await filterSourceDuplicates([release.poster], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
if (!newPoster) return;
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
console.log(`Fetching poster for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-16 02:33:36 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const metaFile = await fetchPhoto(release.poster, null, `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
|
|
|
const [uniquePoster] = await filterHashDuplicates([metaFile], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
if (!uniquePoster) return;
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const savedPosters = await savePhotos([uniquePoster], release, releaseId, null, true);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
await knex('media').insert(curatePhotoEntries(savedPosters, 'releases', 'poster', releaseId));
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
async function storePhotos(release, releaseId) {
|
|
|
|
if (!release.photos || release.photos.length === 0) {
|
|
|
|
console.warn(`No photos available for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
|
|
|
return;
|
|
|
|
}
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const newPhotos = await filterSourceDuplicates(release.photos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
if (newPhotos.length === 0) return;
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
console.log(`Fetching ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
|
|
|
|
|
|
|
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), {
|
2019-12-04 20:58:08 +00:00
|
|
|
concurrency: 10,
|
2019-11-11 02:20:00 +00:00
|
|
|
});
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
|
|
|
const savedPhotos = await savePhotos(uniquePhotos, release, releaseId);
|
|
|
|
|
|
|
|
await knex('media').insert(curatePhotoEntries(savedPhotos, 'releases', 'photo', releaseId));
|
|
|
|
|
|
|
|
console.log(`Stored ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
}
|
|
|
|
|
2019-11-19 03:36:15 +00:00
|
|
|
async function storeTrailer(release, releaseId) {
|
2019-12-07 03:17:14 +00:00
|
|
|
// support scrapers supplying multiple qualities
|
|
|
|
const trailer = Array.isArray(release.trailer) ? release.trailer[0] : release.trailer;
|
|
|
|
|
|
|
|
if (!trailer || !trailer.src) {
|
2019-11-19 03:36:15 +00:00
|
|
|
console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
|
2019-11-16 02:33:36 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-11-19 03:36:15 +00:00
|
|
|
console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 03:17:14 +00:00
|
|
|
const { pathname } = new URL(trailer.src);
|
|
|
|
const mimetype = trailer.type || mime.getType(pathname);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
2019-12-07 03:17:14 +00:00
|
|
|
const res = await bhttp.get(trailer.src);
|
|
|
|
const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
2019-11-11 02:20:00 +00:00
|
|
|
|
|
|
|
await Promise.all([
|
|
|
|
fs.writeFile(path.join(config.media.path, filepath), res.body),
|
|
|
|
knex('media').insert({
|
|
|
|
path: filepath,
|
|
|
|
mime: mimetype,
|
2019-12-07 03:17:14 +00:00
|
|
|
source: trailer.src,
|
2019-11-11 02:20:00 +00:00
|
|
|
domain: 'releases',
|
2019-11-19 03:36:15 +00:00
|
|
|
target_id: releaseId,
|
2019-11-11 02:20:00 +00:00
|
|
|
role: 'trailer',
|
2019-12-07 03:17:14 +00:00
|
|
|
quality: trailer.quality || null,
|
2019-11-11 02:20:00 +00:00
|
|
|
}),
|
|
|
|
]);
|
|
|
|
}
|
|
|
|
|
2019-11-20 03:53:36 +00:00
|
|
|
async function storeAvatars(profile, actor) {
|
|
|
|
if (!profile.avatars || profile.avatars.length === 0) {
|
|
|
|
console.warn(`No avatars available for '${profile.name}'`);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const newPhotos = await filterSourceDuplicates(profile.avatars, 'actors', ['avatar', 'photo'], actor.name);
|
2019-11-20 03:53:36 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
if (newPhotos.length === 0) return;
|
2019-11-21 03:05:32 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
console.log(`Fetching ${newPhotos.length} avatars for '${actor.name}'`);
|
2019-11-20 03:53:36 +00:00
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), {
|
2019-12-04 20:58:08 +00:00
|
|
|
concurrency: 10,
|
2019-11-20 03:53:36 +00:00
|
|
|
});
|
|
|
|
|
2019-12-07 02:59:55 +00:00
|
|
|
const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name);
|
|
|
|
const [savedPhotos, avatarEntry] = await Promise.all([
|
|
|
|
savePhotos(uniquePhotos, null, null, actor.slug),
|
|
|
|
knex('media').where({
|
2019-11-20 03:53:36 +00:00
|
|
|
target_id: actor.id,
|
2019-12-07 02:59:55 +00:00
|
|
|
domain: 'actors',
|
|
|
|
role: 'avatar',
|
|
|
|
}).first(),
|
|
|
|
]);
|
|
|
|
|
|
|
|
// if no avatar entry is present, curatePhotoEntries will store the first photo as avatar
|
|
|
|
await knex('media').insert(curatePhotoEntries(savedPhotos, 'actors', 'photo', actor.id, !avatarEntry));
|
2019-11-20 03:53:36 +00:00
|
|
|
}
|
|
|
|
|
2019-11-11 02:20:00 +00:00
|
|
|
module.exports = {
|
2019-11-20 03:53:36 +00:00
|
|
|
createActorMediaDirectory,
|
|
|
|
createReleaseMediaDirectory,
|
|
|
|
storeAvatars,
|
2019-11-11 02:20:00 +00:00
|
|
|
storePoster,
|
|
|
|
storePhotos,
|
|
|
|
storeTrailer,
|
|
|
|
};
|