DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Batching and logging S3 media deletion.

This commit is contained in:
DebaucheryLibrarian 2025-02-26 06:33:51 +01:00
parent 6d1faf4118
commit 6f8815b47b
1 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/media.js
View File

@ -13,6 +13,7 @@ const mime = require('mime');
const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp');
const blake2 = require('blake2');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue');
const { Upload } = require('@aws-sdk/lib-storage');
@ -998,27 +999,33 @@ async function associateAvatars(profiles) {
return profilesWithAvatarId;
}
async function deleteS3Objects(media) {
const objects = media
.map((item) => [
{ Key: item.path },
{ Key: item.thumbnail },
{ Key: item.lazy },
])
.flat()
.filter((item) => item.Key);
async function deleteS3Objects(allMedia) {
const statuses = await chunk(allMedia).reduce(async (chain, media) => {
const acc = await chain;
const status = await s3.deleteObjects({
Bucket: config.s3.bucket,
Delete: {
Objects: objects,
Quiet: false,
},
const objects = media
.map((item) => [
{ Key: item.path },
{ Key: item.thumbnail },
{ Key: item.lazy },
])
.flat()
.filter((item) => item.Key);
const status = await s3.deleteObjects({
Bucket: config.s3.bucket,
Delete: {
Objects: objects,
Quiet: false,
},
});
logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`);
return acc.concat(status);
});
logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`);
return status;
return statuses;
}
async function flushOrphanedMedia() {
@ -1052,6 +1059,8 @@ async function flushOrphanedMedia() {
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete();
await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (argv.flushMediaFiles) {
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),