DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Indexed media table foreign keys for improved delete performance. Staged media flushing.

This commit is contained in:
DebaucheryLibrarian 2025-03-04 03:16:07 +01:00
parent 39477e4561
commit d82fc704c1
2 changed files with 94 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
migrations/20250304025013_media_indexes.js Normal file
View File

@ -0,0 +1,53 @@
exports.up = async (knex) => {
await knex.schema.alterTable('media', (table) => table.index('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.index('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.index('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.index('media_id'));
};
exports.down = async (knex) => {
await knex.schema.alterTable('media', (table) => table.dropIndex('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.dropIndex('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.dropIndex('media_id'));
};

70
src/media.js
View File

@ -1032,41 +1032,44 @@ async function flushOrphanedMedia(stage = 1) {
logger.info(`Flushing orphaned media, stage ${stage}`); logger.info(`Flushing orphaned media, stage ${stage}`);
const orphanedMedia = await knex('media') const orphanedMedia = await knex('media')
.select('id', 'path', 'thumbnail', 'lazy', 'is_s3')
.where('is_sfw', false) .where('is_sfw', false)
.whereNotExists( .whereNotExists(
knex knex.from(
.from( knex('tags_posters')
knex('tags_posters') .select('media_id')
.select('media_id') .unionAll(
.unionAll( knex('tags_photos').select('media_id'),
knex('tags_photos').select('media_id'), knex('releases_posters').select('media_id'),
knex('releases_posters').select('media_id'), knex('releases_photos').select('media_id'),
knex('releases_photos').select('media_id'), knex('releases_caps').select('media_id'),
knex('releases_caps').select('media_id'), knex('releases_covers').select('media_id'),
knex('releases_covers').select('media_id'), knex('releases_trailers').select('media_id'),
knex('releases_trailers').select('media_id'), knex('releases_teasers').select('media_id'),
knex('releases_teasers').select('media_id'), knex('movies_covers').select('media_id'),
knex('movies_covers').select('media_id'), knex('movies_trailers').select('media_id'),
knex('movies_trailers').select('media_id'), knex('movies_teasers').select('media_id'),
knex('movies_teasers').select('media_id'), knex('actors').select(knex.raw('avatar_media_id as media_id')),
knex('actors').select(knex.raw('avatar_media_id as media_id')), knex('actors_profiles').select(knex.raw('avatar_media_id as media_id')),
knex('actors_profiles').select(knex.raw('avatar_media_id as media_id')), knex('actors_photos').select('media_id'),
knex('actors_photos').select('media_id'), knex('actors_avatars').select('media_id'),
knex('actors_avatars').select('media_id'), knex('chapters_photos').select('media_id'),
knex('chapters_photos').select('media_id'), knex('chapters_posters').select('media_id'),
knex('chapters_posters').select('media_id'), )
) .as('associations'),
.as('associations'), )
) .whereRaw('associations.media_id = media.id'),
.whereRaw('associations.media_id = media.id')
.limit(config.media.flushWindow),
) )
.returning(['media.id', 'media.is_s3', 'media.path', 'media.thumbnail', 'media.lazy']) .limit(config.media.flushWindow);
.delete(); // .delete();
logger.info(`Found ${orphanedMedia.length} orphaned media entries in stage ${stage}`); logger.info(`Found ${orphanedMedia.length} orphaned media entries in stage ${stage}`);
await fs.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4)); if (orphanedMedia.length === 0) {
return;
}
await fs.promises.writeFile(`log/deletedmedia_${format(new Date(), 'yyyy-MM-dd_hh:mm:ss')}.log`, JSON.stringify(orphanedMedia, null, 4));
if (argv.flushMediaFiles) { if (argv.flushMediaFiles) {
await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([ await Promise.all(orphanedMedia.filter((media) => !media.is_s3).map((media) => Promise.all([
@ -1086,11 +1089,20 @@ async function flushOrphanedMedia(stage = 1) {
try { try {
await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true }); await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
await fsPromises.mkdir(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Cleared temporary media directory'); logger.info('Cleared temporary media directory');
} catch (error) { } catch (error) {
logger.warn(`Failed to clear temporary media directory: ${error.message}`); logger.warn(`Failed to clear temporary media directory: ${error.message}`);
} }
// delete database entries last, so in case of failure we don't end up with unrecoverably orphaned external media
const deletedCount = await knex('media')
.whereIn('id', orphanedMedia.map((media) => media.id))
.delete();
logger.info(`Deleted ${deletedCount} orphaned media entries from database`);
if (orphanedMedia.length > 0 && orphanedMedia.length >= config.media.flushWindow) { if (orphanedMedia.length > 0 && orphanedMedia.length >= config.media.flushWindow) {
await flushOrphanedMedia(stage + 1); await flushOrphanedMedia(stage + 1);
} }