From 2b5aac7633242b00679d7fb7e6b57352b8b56ca4 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 23 Feb 2021 01:30:38 +0100 Subject: [PATCH] Fixed S3 display support for movies. --- assets/components/releases/banner.vue | 2 +- assets/components/releases/release.vue | 2 +- assets/js/fragments.js | 1 + assets/js/releases/actions.js | 14 +++++++++++ migrations/20190325001339_releases.js | 2 +- src/media.js | 33 +++++++++++++++++++++++--- src/scrapers/gamma.js | 2 -- 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/assets/components/releases/banner.vue b/assets/components/releases/banner.vue index 947e456d..80fb50e4 100644 --- a/assets/components/releases/banner.vue +++ b/assets/components/releases/banner.vue @@ -59,7 +59,7 @@ diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index 07379d0e..3a88752d 100644 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -84,7 +84,7 @@ {{ movie.title }} diff --git a/assets/js/fragments.js b/assets/js/fragments.js index 7845898f..fd4de842 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -315,6 +315,7 @@ const releaseFragment = ` path thumbnail lazy + isS3 comment sfw: sfwMedia { id diff --git a/assets/js/releases/actions.js b/assets/js/releases/actions.js index 817410eb..384d57fc 100644 --- a/assets/js/releases/actions.js +++ b/assets/js/releases/actions.js @@ -103,6 +103,7 @@ function initReleasesActions(store, router) { path thumbnail lazy + isS3 sfw: sfwMedia { id path @@ -157,6 +158,16 @@ function initReleasesActions(store, router) { path thumbnail lazy + isS3 + } + } + posters: moviesPosterByMovieId { + media { + id + path + thumbnail + lazy + isS3 } } covers: moviesCovers { @@ -165,12 +176,14 @@ function initReleasesActions(store, router) { path thumbnail lazy + isS3 } } trailer: moviesTrailerByMovieId { media { id path + isS3 } } scenes: moviesScenes { @@ -189,6 +202,7 @@ function initReleasesActions(store, router) { path thumbnail lazy + isS3 comment sfw: sfwMedia { id diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index c1b9d90d..62696b39 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -901,7 +901,7 @@ exports.up = knex => Promise.resolve() .references('id') .inTable('media'); - table.unique(['movie_id', 'media_id']); + table.unique('movie_id'); })) .then(() => knex.schema.createTable('clips', (table) => { table.increments('id', 16); diff --git a/src/media.js b/src/media.js index 4c3c4289..72d9db77 100644 --- a/src/media.js +++ b/src/media.js @@ -26,7 +26,7 @@ const { get } = require('./utils/qu'); const pipeline = util.promisify(stream.pipeline); const streamQueue = taskQueue(); -const endpoint = new AWS.Endpoint('s3.wasabisys.com'); +const endpoint = new AWS.Endpoint('s3.eu-central-1.wasabisys.com'); const s3 = new AWS.S3({ // region: 'eu-central-1', @@ -869,6 +869,29 @@ async function associateAvatars(profiles) { return profilesWithAvatarIds; } +async function deleteS3Objects(media) { + const objects = media + .map(item => [ + { Key: item.path }, + { Key: item.thumbnail }, + { Key: item.lazy }, + ]) + .flat() + .filter(item => item.Key); + + const status = await s3.deleteObjects({ + Bucket: config.s3.bucket, + Delete: { + Objects: objects, + Quiet: false, + }, + }).promise(); + + logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`); + + return status; +} + async function flushOrphanedMedia() { const orphanedMedia = await knex('media') .where('is_sfw', false) @@ -896,10 +919,10 @@ async function flushOrphanedMedia() { ) .whereRaw('associations.media_id = media.id'), ) - .returning(['media.path', 'media.thumbnail', 'media.lazy']) + .returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) .delete(); - await Promise.all(orphanedMedia.map(media => Promise.all([ + await Promise.all(orphanedMedia.filter(media => !media.is_s3).map(media => Promise.all([ media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }), media.thumbnail && fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }), media.lazy && fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }), @@ -907,6 +930,10 @@ async function flushOrphanedMedia() { logger.info(`Removed ${orphanedMedia.length} media files from database and storage`); + if (config.s3.enabled) { + await deleteS3Objects(orphanedMedia.filter(media => media.is_s3)); + } + await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true }); logger.info('Cleared temporary media directory'); diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 336b6528..5ee2fb7f 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -200,8 +200,6 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) { ]; } - console.log(release); - return release; }); }