DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Batching and logging S3 media deletion.

This commit is contained in:
DebaucheryLibrarian 2025-02-26 06:33:51 +01:00
parent 6d1faf4118
commit 6f8815b47b
1 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/media.js
View File

@ -13,6 +13,7 @@ const mime = require('mime');
const ffmpeg = require('fluent-ffmpeg'); const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp'); const sharp = require('sharp');
const blake2 = require('blake2'); const blake2 = require('blake2');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue'); const taskQueue = require('promise-task-queue');
const { Upload } = require('@aws-sdk/lib-storage'); const { Upload } = require('@aws-sdk/lib-storage');
@ -998,7 +999,10 @@ async function associateAvatars(profiles) {
return profilesWithAvatarId; return profilesWithAvatarId;
} }
async function deleteS3Objects(media) { async function deleteS3Objects(allMedia) {
const statuses = await chunk(allMedia).reduce(async (chain, media) => {
const acc = await chain;
const objects = media const objects = media
.map((item) => [ .map((item) => [
{ Key: item.path }, { Key: item.path },
@ -1018,7 +1022,10 @@ async function deleteS3Objects(media) {
logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`); logger.info(`Removed ${status.Deleted.length} media files from S3 bucket '${config.s3.bucket}', ${status.Errors.length} errors`);
return status; return acc.concat(status);
});
return statuses;
} }
async function flushOrphanedMedia() { async function flushOrphanedMedia() {
@ -1052,6 +1059,8 @@ async function flushOrphanedMedia() {
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) .returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete(); .delete();
await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (argv.flushMediaFiles) { if (argv.flushMediaFiles) {
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([ await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }), media.path && fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),