From d82fc704c174488c870940fdaa905c1472333a84 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 4 Mar 2025 03:16:07 +0100 Subject: [PATCH] Indexed media table foreign keys for improved delete performance. Staged media flushing. --- migrations/20250304025013_media_indexes.js | 53 ++++++++++++++++ src/media.js | 70 +++++++++++++--------- 2 files changed, 94 insertions(+), 29 deletions(-) create mode 100644 migrations/20250304025013_media_indexes.js diff --git a/migrations/20250304025013_media_indexes.js b/migrations/20250304025013_media_indexes.js new file mode 100644 index 00000000..a03a9cba --- /dev/null +++ b/migrations/20250304025013_media_indexes.js @@ -0,0 +1,53 @@ +exports.up = async (knex) => { + await knex.schema.alterTable('media', (table) => table.index('sfw_media_id')); + await knex.schema.alterTable('actors_profiles', (table) => table.index('avatar_media_id')); + await knex.schema.alterTable('actors_avatars', (table) => table.index('media_id')); + await knex.schema.alterTable('actors_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('chapters_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('chapters_posters', (table) => table.index('media_id')); + await knex.schema.alterTable('movies_covers', (table) => table.index('media_id')); + await knex.schema.alterTable('movies_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('movies_posters', (table) => table.index('media_id')); + await knex.schema.alterTable('movies_teasers', (table) => table.index('media_id')); + await knex.schema.alterTable('movies_trailers', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_caps', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_covers', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_posters', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_teasers', (table) => table.index('media_id')); + await knex.schema.alterTable('releases_trailers', (table) => table.index('media_id')); + await knex.schema.alterTable('series_covers', (table) => table.index('media_id')); + await knex.schema.alterTable('series_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('series_posters', (table) => table.index('media_id')); + await knex.schema.alterTable('series_teasers', (table) => table.index('media_id')); + await knex.schema.alterTable('series_trailers', (table) => table.index('media_id')); + await knex.schema.alterTable('tags_photos', (table) => table.index('media_id')); + await knex.schema.alterTable('tags_posters', (table) => table.index('media_id')); +}; + +exports.down = async (knex) => { + await knex.schema.alterTable('media', (table) => table.dropIndex('sfw_media_id')); + await knex.schema.alterTable('actors_profiles', (table) => table.dropIndex('avatar_media_id')); + await knex.schema.alterTable('actors_avatars', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('actors_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('chapters_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('chapters_posters', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('movies_covers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('movies_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('movies_posters', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('movies_teasers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('movies_trailers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_caps', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_covers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_posters', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_teasers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('releases_trailers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('series_covers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('series_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('series_posters', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('series_teasers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('series_trailers', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('tags_photos', (table) => table.dropIndex('media_id')); + await knex.schema.alterTable('tags_posters', (table) => table.dropIndex('media_id')); +}; diff --git a/src/media.js b/src/media.js index 9e5894b9..4f7be4f1 100755 --- a/src/media.js +++ b/src/media.js @@ -1032,41 +1032,44 @@ async function flushOrphanedMedia(stage = 1) { logger.info(`Flushing orphaned media, stage ${stage}`); const orphanedMedia = await knex('media') + .select('id', 'path', 'thumbnail', 'lazy', 'is_s3') .where('is_sfw', false) .whereNotExists( - knex - .from( - knex('tags_posters') - .select('media_id') - .unionAll( - knex('tags_photos').select('media_id'), - knex('releases_posters').select('media_id'), - knex('releases_photos').select('media_id'), - knex('releases_caps').select('media_id'), - knex('releases_covers').select('media_id'), - knex('releases_trailers').select('media_id'), - knex('releases_teasers').select('media_id'), - knex('movies_covers').select('media_id'), - knex('movies_trailers').select('media_id'), - knex('movies_teasers').select('media_id'), - knex('actors').select(knex.raw('avatar_media_id as media_id')), - knex('actors_profiles').select(knex.raw('avatar_media_id as media_id')), - knex('actors_photos').select('media_id'), - knex('actors_avatars').select('media_id'), - knex('chapters_photos').select('media_id'), - knex('chapters_posters').select('media_id'), - ) - .as('associations'), - ) - .whereRaw('associations.media_id = media.id') - .limit(config.media.flushWindow), + knex.from( + knex('tags_posters') + .select('media_id') + .unionAll( + knex('tags_photos').select('media_id'), + knex('releases_posters').select('media_id'), + knex('releases_photos').select('media_id'), + knex('releases_caps').select('media_id'), + knex('releases_covers').select('media_id'), + knex('releases_trailers').select('media_id'), + knex('releases_teasers').select('media_id'), + knex('movies_covers').select('media_id'), + knex('movies_trailers').select('media_id'), + knex('movies_teasers').select('media_id'), + knex('actors').select(knex.raw('avatar_media_id as media_id')), + knex('actors_profiles').select(knex.raw('avatar_media_id as media_id')), + knex('actors_photos').select('media_id'), + knex('actors_avatars').select('media_id'), + knex('chapters_photos').select('media_id'), + knex('chapters_posters').select('media_id'), + ) + .as('associations'), + ) + .whereRaw('associations.media_id = media.id'), ) - .returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) - .delete(); + .limit(config.media.flushWindow); + // .delete(); logger.info(`Found ${orphanedMedia.length} orphaned media entries in stage ${stage}`); - await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4)); + if (orphanedMedia.length === 0) { + return; + } + + await fs.promises.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4)); if (argv.flushMediaFiles) { await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([ @@ -1086,11 +1089,20 @@ async function flushOrphanedMedia(stage = 1) { try { await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true }); + await fsPromises.mkdir(path.join(config.media.path, 'temp'), { recursive: true }); + logger.info('Cleared temporary media directory'); } catch (error) { logger.warn(`Failed to clear temporary media directory: ${error.message}`); } + // delete database entries last, so in case of failure we don't end up with unrecoverably orphaned external media + const deletedCount = await knex('media') + .whereIn('id', orphanedMedia.map((media) => media.id)) + .delete(); + + logger.info(`Deleted ${deletedCount} orphaned media entries from database`); + if (orphanedMedia.length > 0 && orphanedMedia.length >= config.media.flushWindow) { await flushOrphanedMedia(stage + 1); }