Refactored media module to generalize avatar and poster storage into photo storage.
This commit is contained in:
190
src/media.js
190
src/media.js
@@ -19,7 +19,7 @@ function getHash(buffer) {
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
function pluckPhotos(photos, release, specifiedLimit) {
|
||||
function pluckPhotos(photos, specifiedLimit) {
|
||||
const limit = specifiedLimit || config.media.limit;
|
||||
|
||||
if (photos.length <= limit) {
|
||||
@@ -34,7 +34,7 @@ function pluckPhotos(photos, release, specifiedLimit) {
|
||||
return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
|
||||
}
|
||||
|
||||
async function getThumbnail(buffer) {
|
||||
async function createThumbnail(buffer) {
|
||||
return sharp(buffer)
|
||||
.resize({
|
||||
height: config.media.thumbnailSize,
|
||||
@@ -43,25 +43,14 @@ async function getThumbnail(buffer) {
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function createReleaseMediaDirectory(release, releaseId) {
|
||||
if (release.poster || (release.photos && release.photos.length) || release.trailer) {
|
||||
await fs.mkdir(
|
||||
path.join(config.media.path, 'releases', release.site.network.slug, release.site.slug, releaseId.toString()),
|
||||
{ recursive: true },
|
||||
);
|
||||
}
|
||||
async function createMediaDirectory(domain, subpath) {
|
||||
const filepath = path.join(config.media.path, domain, subpath);
|
||||
|
||||
await fs.mkdir(filepath, { recursive: true });
|
||||
return filepath;
|
||||
}
|
||||
|
||||
async function createActorMediaDirectory(profile, actor) {
|
||||
if (profile.avatars && profile.avatars.length) {
|
||||
await fs.mkdir(
|
||||
path.join(config.media.path, 'actors', actor.slug),
|
||||
{ recursive: true },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId, setAvatar = false) {
|
||||
function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId) {
|
||||
return files.map((file, index) => ({
|
||||
path: file.filepath,
|
||||
thumbnail: file.thumbpath,
|
||||
@@ -71,7 +60,7 @@ function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId
|
||||
index,
|
||||
domain,
|
||||
target_id: targetId,
|
||||
role: setAvatar && index === 0 ? 'avatar' : role,
|
||||
role: file.role || role,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -79,8 +68,8 @@ function curatePhotoEntries(files, domain = 'releases', role = 'photo', targetId
|
||||
async function filterSourceDuplicates(photos, domains = ['releases'], roles = ['photo'], identifier) {
|
||||
const photoSourceEntries = await knex('media')
|
||||
.whereIn('source', photos.flat())
|
||||
.whereIn('domain', [].concat(domains))
|
||||
.whereIn('role', [].concat(roles)); // accept string argument
|
||||
.whereIn('domain', domains)
|
||||
.whereIn('role', roles); // accept string argument
|
||||
|
||||
const photoSources = new Set(photoSourceEntries.map(photo => photo.source));
|
||||
const newPhotos = photos.filter(source => (Array.isArray(source) // fallbacks provided?
|
||||
@@ -156,18 +145,22 @@ async function fetchPhoto(photoUrl, index, identifier, attempt = 1) {
|
||||
}
|
||||
}
|
||||
|
||||
async function savePhotos(files, release, releaseId, actorSlug, isPoster = false) {
|
||||
async function savePhotos(files, {
|
||||
domain = 'releases',
|
||||
subpath,
|
||||
role = 'photo',
|
||||
naming = 'index',
|
||||
}) {
|
||||
return Promise.map(files, async (file, index) => {
|
||||
const timestamp = new Date().getTime();
|
||||
const thumbnail = await getThumbnail(file.photo);
|
||||
const thumbnail = await createThumbnail(file.photo);
|
||||
|
||||
const filepath = actorSlug
|
||||
? path.join('actors', actorSlug, `${timestamp + index}.${file.extension}`)
|
||||
: path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}.${file.extension}`);
|
||||
const filename = naming === 'index'
|
||||
? `${file.role || role}-${index}`
|
||||
: `${timestamp + index}`;
|
||||
|
||||
const thumbpath = actorSlug
|
||||
? path.join('actors', actorSlug, `${timestamp + index}_thumb.${file.extension}`)
|
||||
: path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${isPoster ? 'poster' : index + 1}_thumb.${file.extension}`);
|
||||
const filepath = path.join(domain, subpath, `${filename}.${file.extension}`);
|
||||
const thumbpath = path.join(domain, subpath, `${filename}_thumb.${file.extension}`);
|
||||
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(config.media.path, filepath), file.photo),
|
||||
@@ -183,75 +176,85 @@ async function savePhotos(files, release, releaseId, actorSlug, isPoster = false
|
||||
});
|
||||
}
|
||||
|
||||
async function storePoster(release, releaseId) {
|
||||
if (!release.poster) {
|
||||
console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
|
||||
return;
|
||||
}
|
||||
const [newPoster] = await filterSourceDuplicates([release.poster], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
|
||||
if (!newPoster) return;
|
||||
|
||||
console.log(`Fetching poster for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
|
||||
const metaFile = await fetchPhoto(release.poster, null, `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
const [uniquePoster] = await filterHashDuplicates([metaFile], 'releases', 'poster', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
|
||||
if (!uniquePoster) return;
|
||||
|
||||
const savedPosters = await savePhotos([uniquePoster], release, releaseId, null, true);
|
||||
|
||||
await knex('media').insert(curatePhotoEntries(savedPosters, 'releases', 'poster', releaseId));
|
||||
}
|
||||
|
||||
|
||||
async function storePhotos(release, releaseId) {
|
||||
if (!release.photos || release.photos.length === 0) {
|
||||
console.warn(`No photos available for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
async function storePhotos(photos, {
|
||||
domain = 'releases',
|
||||
role = 'photo',
|
||||
naming = 'index',
|
||||
targetId,
|
||||
subpath,
|
||||
primaryRole, // role to assign to first photo if not already in database, used mainly for avatars
|
||||
}, identifier) {
|
||||
if (!photos || photos.length === 0) {
|
||||
console.warn(`No ${role}s available for ${identifier}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const pluckedPhotos = pluckPhotos(release.photos, release);
|
||||
const newPhotos = await filterSourceDuplicates(pluckedPhotos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
const pluckedPhotos = pluckPhotos(photos);
|
||||
const roles = primaryRole ? [role, primaryRole] : [role];
|
||||
|
||||
const newPhotos = await filterSourceDuplicates(pluckedPhotos, [domain], roles, identifier);
|
||||
|
||||
if (newPhotos.length === 0) return;
|
||||
|
||||
console.log(`Fetching ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
console.log(`Fetching ${newPhotos.length} ${role}s for ${identifier}`);
|
||||
|
||||
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), {
|
||||
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, identifier), {
|
||||
concurrency: 10,
|
||||
}).filter(photo => photo);
|
||||
|
||||
console.log(metaFiles);
|
||||
const [uniquePhotos, primaryPhoto] = await Promise.all([
|
||||
filterHashDuplicates(metaFiles, [domain], roles, identifier),
|
||||
primaryRole
|
||||
? await knex('media')
|
||||
.where('domain', domain)
|
||||
.where('target_id', targetId)
|
||||
.where('role', primaryRole)
|
||||
.first()
|
||||
: null,
|
||||
]);
|
||||
|
||||
const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
const savedPhotos = await savePhotos(uniquePhotos, release, releaseId);
|
||||
if (primaryRole && !primaryPhoto) {
|
||||
uniquePhotos[0].role = primaryRole;
|
||||
}
|
||||
|
||||
const curatedPhotoEntries = curatePhotoEntries(savedPhotos, 'releases', 'photo', releaseId);
|
||||
const savedPhotos = await savePhotos(uniquePhotos, {
|
||||
domain,
|
||||
role,
|
||||
targetId,
|
||||
subpath,
|
||||
naming,
|
||||
});
|
||||
|
||||
const curatedPhotoEntries = curatePhotoEntries(savedPhotos, domain, role, targetId);
|
||||
|
||||
await knex('media').insert(curatedPhotoEntries);
|
||||
|
||||
console.log(`Stored ${newPhotos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
console.log(`Stored ${newPhotos.length} ${role}s for ${identifier}`);
|
||||
}
|
||||
|
||||
async function storeTrailer(release, releaseId) {
|
||||
async function storeTrailer(trailers, {
|
||||
domain = 'releases',
|
||||
role = 'trailer',
|
||||
targetId,
|
||||
subpath,
|
||||
}, identifier) {
|
||||
// support scrapers supplying multiple qualities
|
||||
const trailer = Array.isArray(release.trailer)
|
||||
? (release.trailer.find(trailerX => [1080, 720].includes(trailerX.quality) || release.trailer[0]))
|
||||
: release.trailer;
|
||||
const trailer = Array.isArray(trailers)
|
||||
? trailers.find(trailerX => [1080, 720].includes(trailerX.quality)) || trailers[0]
|
||||
: trailers;
|
||||
|
||||
if (!trailer || !trailer.src) {
|
||||
console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
|
||||
console.warn(`No trailer available for ${identifier}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`);
|
||||
console.log(`Storing trailer for ${identifier}`);
|
||||
|
||||
const { pathname } = new URL(trailer.src);
|
||||
const mimetype = trailer.type || mime.getType(pathname);
|
||||
|
||||
const res = await bhttp.get(trailer.src);
|
||||
const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
||||
const filepath = path.join('releases', subpath, `trailer${trailer.quality ? `_${trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
||||
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(config.media.path, filepath), res.body),
|
||||
@@ -259,49 +262,24 @@ async function storeTrailer(release, releaseId) {
|
||||
path: filepath,
|
||||
mime: mimetype,
|
||||
source: trailer.src,
|
||||
domain: 'releases',
|
||||
target_id: releaseId,
|
||||
role: 'trailer',
|
||||
domain,
|
||||
target_id: targetId,
|
||||
role,
|
||||
quality: trailer.quality || null,
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
async function storeAvatars(profile, actor) {
|
||||
if (!profile.avatars || profile.avatars.length === 0) {
|
||||
console.warn(`No avatars available for '${profile.name}'`);
|
||||
return;
|
||||
}
|
||||
|
||||
const newPhotos = await filterSourceDuplicates(profile.avatars, 'actors', ['avatar', 'photo'], actor.name);
|
||||
|
||||
if (newPhotos.length === 0) return;
|
||||
|
||||
console.log(`Fetching ${newPhotos.length} avatars for '${actor.name}'`);
|
||||
|
||||
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), {
|
||||
concurrency: 10,
|
||||
}).filter(photo => photo);
|
||||
|
||||
const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name);
|
||||
const [savedPhotos, avatarEntry] = await Promise.all([
|
||||
savePhotos(uniquePhotos, null, null, actor.slug),
|
||||
knex('media').where({
|
||||
target_id: actor.id,
|
||||
domain: 'actors',
|
||||
role: 'avatar',
|
||||
}).first(),
|
||||
]);
|
||||
|
||||
// if no avatar entry is present, curatePhotoEntries will store the first photo as avatar
|
||||
await knex('media').insert(curatePhotoEntries(savedPhotos, 'actors', 'photo', actor.id, !avatarEntry));
|
||||
async function findAvatar(actorId, domain = 'actors') {
|
||||
return knex('media')
|
||||
.where('domain', domain)
|
||||
.where('target_id', actorId)
|
||||
.where('role', 'avatar');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createActorMediaDirectory,
|
||||
createReleaseMediaDirectory,
|
||||
storeAvatars,
|
||||
storePoster,
|
||||
createMediaDirectory,
|
||||
findAvatar,
|
||||
storePhotos,
|
||||
storeTrailer,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user