DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Indexed media table foreign keys for improved delete performance. Staged media flushing.

This commit is contained in:
DebaucheryLibrarian 2025-03-04 03:16:07 +01:00
parent 39477e4561
commit d82fc704c1
2 changed files with 94 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
migrations/20250304025013_media_indexes.js Normal file
View File

@ -0,0 +1,53 @@
exports.up = async (knex) => {
await knex.schema.alterTable('media', (table) => table.index('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.index('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.index('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.index('media_id'));
};
exports.down = async (knex) => {
await knex.schema.alterTable('media', (table) => table.dropIndex('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.dropIndex('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.dropIndex('media_id'));
};

26
src/media.js
View File

@ -1032,10 +1032,10 @@ async function flushOrphanedMedia(stage = 1) {
logger.info(`Flushing orphaned media, stage ${stage}`);
const orphanedMedia = await knex('media')
.select('id', 'path', 'thumbnail', 'lazy', 'is_s3')
.where('is_sfw', false)
.whereNotExists(
knex
.from(
knex.from(
knex('tags_posters')
.select('media_id')
.unionAll(
@ -1058,15 +1058,18 @@ async function flushOrphanedMedia(stage = 1) {
)
.as('associations'),
)
.whereRaw('associations.media_id = media.id')
.limit(config.media.flushWindow),
.whereRaw('associations.media_id = media.id'),
)
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy'])
.delete();
.limit(config.media.flushWindow);
// .delete();
logger.info(`Found ${orphanedMedia.length} orphaned media entries in stage ${stage}`);
await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (orphanedMedia.length === 0) {
return;
}
await fs.promises.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (argv.flushMediaFiles) {
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
@ -1086,11 +1089,20 @@ async function flushOrphanedMedia(stage = 1) {
try {
await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
await fsPromises.mkdir(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Cleared temporary media directory');
} catch (error) {
logger.warn(`Failed to clear temporary media directory: ${error.message}`);
}
// delete database entries last, so in case of failure we don't end up with unrecoverably orphaned external media
const deletedCount = await knex('media')
.whereIn('id', orphanedMedia.map((media) => media.id))
.delete();
logger.info(`Deleted ${deletedCount} orphaned media entries from database`);
if (orphanedMedia.length > 0 && orphanedMedia.length >= config.media.flushWindow) {
await flushOrphanedMedia(stage + 1);
}