DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Batching and logging S3 media deletion.

This commit is contained in:
DebaucheryLibrarian 2025-02-26 06:33:51 +01:00
parent 6d1faf4118
commit 6f8815b47b
1 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/media.js
View File

@ -13,6 +13,7 @@ const mime = require('mime');
const ffmpeg = require('fluent-ffmpeg'); const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp'); const sharp = require('sharp');
const blake2 = require('blake2'); const blake2 = require('blake2');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue'); const taskQueue = require('promise-task-queue');
const { Upload } = require('@aws-sdk/lib-storage'); const { Upload } = require('@aws-sdk/lib-storage');
@ -998,27 +999,33 @@ async function associateAvatars(profiles) {
return profilesWithAvatarId; return profilesWithAvatarId;
} }
async function deleteS3Objects(media) { async function deleteS3Objects(allMedia) {
const objects = media const statuses = await chunk(allMedia).reduce(async (chain, media) => {
.map((item) => [ const acc = await chain;
{ Key: item.path },
{ Key: item.thumbnail },
{ Key: item.lazy },
])
.flat()
.filter((item) => item.Key);
const status = await s3.deleteObjects({ const objects = media
Bucket: config.s3.bucket, .map((item) => [
Delete: { { Key: item.path },
Objects: objects, { Key: item.thumbnail },
Quiet: false, { Key: item.lazy },
}, ])
.flat()
.filter((item) => item.Key);
const status = await s3.deleteObjects({
Bucket: config.s3.bucket,
Delete: {
Objects: objects,
Quiet: false,
},
});
logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`);
return acc.concat(status);
}); });
logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`); return statuses;
return status;
} }
async function flushOrphanedMedia() { async function flushOrphanedMedia() {
@ -1052,6 +1059,8 @@ async function flushOrphanedMedia() {
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) .returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete(); .delete();
await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (argv.flushMediaFiles) { if (argv.flushMediaFiles) {
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([ await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }), media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),