From 0bd7fca87611551e58abb5b849e32e39d84c2b3a Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 25 Oct 2020 00:52:40 +0200 Subject: [PATCH] Added orphaned media flush and batch release flush. --- assets/components/releases/release.vue | 2 +- assets/js/fragments.js | 2 + migrations/20190325001339_releases.js | 57 +++++++++++++++++--------- src/app.js | 19 ++++++++- src/argv.js | 19 ++++++++- src/entities.js | 25 ++++++++--- src/media.js | 42 +++++++++++++++++++ src/releases.js | 44 +++++++++++++++++++- src/scrapers/traxxx.js | 15 +++++++ 9 files changed, 196 insertions(+), 29 deletions(-) create mode 100644 src/scrapers/traxxx.js diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index 8498c690..d248272f 100644 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -193,7 +193,7 @@ :to="`/added/${formatDate(release.createdAt, 'YYYY/MM/DD')}`" :title="`Added on ${formatDate(release.createdAt, 'MMMM D, YYYY HH:mm')}`" class="link added" - >{{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }} + >{{ release.createdBatchId }}: {{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }} diff --git a/assets/js/fragments.js b/assets/js/fragments.js index 151db959..60cddf88 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -185,6 +185,7 @@ const releaseFields = ` comment createdAt url + createdBatchId ${releaseActorsFragment} ${releaseTagsFragment} ${releasePosterFragment} @@ -238,6 +239,7 @@ const releaseFragment = ` createdAt shootId productionDate + createdBatchId productionLocation productionCity productionState diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 7f520176..7658a2a5 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -296,7 +296,8 @@ exports.up = knex => Promise.resolve() table.integer('batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.datetime('updated_at') .defaultTo(knex.fn.now()); @@ -310,7 +311,8 @@ exports.up = knex => Promise.resolve() table.integer('actor_id', 12) .notNullable() .references('id') - .inTable('actors'); + .inTable('actors') + .onDelete('cascade'); table.integer('entity_id', 12) .references('id') @@ -509,7 +511,8 @@ exports.up = knex => Promise.resolve() table.integer('actor_id', 12) .notNullable() .references('id') - .inTable('actors'); + .inTable('actors') + .onDelete('cascade'); table.text('body_slug', 20) .references('slug') @@ -528,7 +531,8 @@ exports.up = knex => Promise.resolve() table.integer('actor_id', 12) .notNullable() .references('id') - .inTable('actors'); + .inTable('actors') + .onDelete('cascade'); table.text('body_slug', 20) .references('slug') @@ -545,7 +549,8 @@ exports.up = knex => Promise.resolve() table.integer('profile_id', 12) .notNullable() .references('id') - .inTable('actors_profiles'); + .inTable('actors_profiles') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -558,7 +563,8 @@ exports.up = knex => Promise.resolve() table.integer('actor_id', 12) .notNullable() .references('id') - .inTable('actors'); + .inTable('actors') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -645,11 +651,13 @@ exports.up = knex => Promise.resolve() table.integer('created_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.integer('updated_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.datetime('created_at') .defaultTo(knex.fn.now()); @@ -664,7 +672,8 @@ exports.up = knex => Promise.resolve() table.integer('actor_id', 12) .notNullable() .references('id') - .inTable('actors'); + .inTable('actors') + .onDelete('cascade'); table.unique(['release_id', 'actor_id']); @@ -809,11 +818,13 @@ exports.up = knex => Promise.resolve() table.integer('created_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.integer('updated_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.datetime('created_at') .defaultTo(knex.fn.now()); @@ -838,7 +849,8 @@ exports.up = knex => Promise.resolve() table.integer('movie_id', 16) .notNullable() .references('id') - .inTable('movies'); + .inTable('movies') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -852,7 +864,8 @@ exports.up = knex => Promise.resolve() .unique() .notNullable() .references('id') - .inTable('movies'); + .inTable('movies') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -865,7 +878,8 @@ exports.up = knex => Promise.resolve() table.integer('release_id', 12) .references('id') .inTable('releases') - .notNullable(); + .notNullable() + .onDelete('cascade'); table.integer('clip', 6); @@ -879,11 +893,13 @@ exports.up = knex => Promise.resolve() table.integer('created_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.integer('updated_batch_id', 12) .references('id') - .inTable('batches'); + .inTable('batches') + .onDelete('cascade'); table.datetime('created_at') .defaultTo(knex.fn.now()); @@ -892,7 +908,8 @@ exports.up = knex => Promise.resolve() table.integer('clip_id', 16) .notNullable() .references('id') - .inTable('clips'); + .inTable('clips') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -905,7 +922,8 @@ exports.up = knex => Promise.resolve() table.integer('clip_id', 16) .notNullable() .references('id') - .inTable('clips'); + .inTable('clips') + .onDelete('cascade'); table.text('media_id', 21) .notNullable() @@ -918,7 +936,8 @@ exports.up = knex => Promise.resolve() table.integer('tag_id', 12) .notNullable() .references('id') - .inTable('tags'); + .inTable('tags') + .onDelete('cascade'); table.integer('clip_id', 16) .notNullable() diff --git a/src/app.js b/src/app.js index 9d1e65aa..9464b5e6 100644 --- a/src/app.js +++ b/src/app.js @@ -11,7 +11,8 @@ const { fetchScenes, fetchMovies } = require('./deep'); const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases'); const { scrapeActors } = require('./actors'); const { flushEntities } = require('./entities'); -const { deleteScenes } = require('./releases'); +const { deleteScenes, deleteMovies, flushBatches } = require('./releases'); +const { flushOrphanedMedia } = require('./media'); const getFileEntries = require('./utils/file-entries'); async function init() { @@ -28,10 +29,26 @@ async function init() { await flushEntities(argv.flushNetworks, argv.flushChannels); } + if (argv.flushBatches) { + await flushBatches(argv.flushBatches); + } + + if (argv.deleteScenes) { + await deleteScenes(argv.deleteScenes); + } + + if (argv.deleteMovies) { + await deleteMovies(argv.deleteMovies); + } + if (argv.delete) { await deleteScenes(argv.delete); } + if (argv.flushOrphanedMedia) { + await flushOrphanedMedia(); + } + const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile); const actorNames = (argv.actors || []).concat(actorsFromFile || []); diff --git a/src/argv.js b/src/argv.js index 61b5afcc..fc37fdc0 100644 --- a/src/argv.js +++ b/src/argv.js @@ -238,6 +238,11 @@ const { argv } = yargs type: 'boolean', default: false, }) + .option('flush-orphaned-media', { + describe: 'Remove all orphaned media items from database and disk.', + type: 'array', + alias: 'flush-media', + }) .option('flush-channels', { describe: 'Delete all scenes and movies from channels.', type: 'array', @@ -248,10 +253,20 @@ const { argv } = yargs type: 'array', alias: 'flush-network', }) - .option('delete', { + .option('flush-batches', { + describe: 'Delete all scenes and movies from batch by ID.', + type: 'array', + alias: 'flush-batch', + }) + .option('delete-scenes', { describe: 'Remove scenes by ID.', type: 'array', - alias: 'remove', + alias: ['delete-scene', 'delete', 'remove', 'remove-scenes', 'remove-scene'], + }) + .option('delete-movies', { + describe: 'Remove movies by ID.', + type: 'array', + alias: ['delete-movie', 'remove-movies', 'remove-movies'], }) .coerce('after', interpretAfter) .coerce('actors-update', interpretAfter); diff --git a/src/entities.js b/src/entities.js index cff8b67a..397e38ad 100644 --- a/src/entities.js +++ b/src/entities.js @@ -6,7 +6,8 @@ const inquirer = require('inquirer'); const logger = require('./logger')(__filename); const argv = require('./argv'); const knex = require('./knex'); -const { deleteScenes } = require('./releases'); +const { deleteScenes, deleteMovies } = require('./releases'); +const { flushOrphanedMedia } = require('./media'); function curateEntity(entity, includeParameters = false) { if (!entity) { @@ -235,7 +236,16 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) { .leftJoin('releases', 'releases.entity_id', 'selected_entities.id') .pluck('releases.id'); - if (sceneIds.length === 0) { + const movieIds = await entityQuery + .clone() + .select('movies.id') + .distinct('movies.id') + .whereNotNull('movies.id') + .from('selected_entities') + .leftJoin('movies', 'movies.entity_id', 'selected_entities.id') + .pluck('movies.id'); + + if (sceneIds.length === 0 && movieIds.length === 0) { logger.info(`No scenes or movies found to remove for ${entitySlugs}`); return; } @@ -243,16 +253,21 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) { const confirmed = await inquirer.prompt([{ type: 'confirm', name: 'flushEntities', - message: `You are about to remove ${sceneIds.length} scenes for ${entitySlugs}. Are you sure?`, + message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies for ${entitySlugs}. Are you sure?`, default: false, }]); if (!confirmed.flushEntities) { - logger.warn(`Confirmation rejected, not flushing scenes for: ${entitySlugs}`); + logger.warn(`Confirmation rejected, not flushing scenes or movies for: ${entitySlugs}`); return; } - await deleteScenes(sceneIds); + await Promise.all([ + deleteScenes(sceneIds), + deleteMovies(movieIds), + ]); + + await flushOrphanedMedia(); } module.exports = { diff --git a/src/media.js b/src/media.js index 378132fe..90c78bda 100644 --- a/src/media.js +++ b/src/media.js @@ -747,7 +747,49 @@ async function associateAvatars(profiles) { return profilesWithAvatarIds; } +async function flushOrphanedMedia() { + const orphanedMedia = await knex('media') + .where('is_sfw', false) + .whereNotExists( + knex + .from( + knex('tags_posters') + .select('media_id') + .unionAll( + knex('tags_photos').select('media_id'), + knex('releases_posters').select('media_id'), + knex('releases_photos').select('media_id'), + knex('releases_trailers').select('media_id'), + knex('releases_teasers').select('media_id'), + knex('movies_covers').select('media_id'), + knex('movies_trailers').select('media_id'), + knex('actors_avatars').select('media_id'), + knex('actors_photos').select('media_id'), + knex('clips_photos').select('media_id'), + knex('clips_posters').select('media_id'), + ) + .as('associations'), + ) + .whereRaw('associations.media_id = media.id'), + ) + .returning(['media.path', 'media.thumbnail', 'media.lazy']) + .delete(); + + await Promise.all(orphanedMedia.map(media => Promise.all([ + fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }), + fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }), + fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }), + ]))); + + logger.info(`Removed ${orphanedMedia.length} media files from database and disk`); + + await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true }); + + logger.info('Removed temporary media directory'); +} + module.exports = { associateAvatars, associateReleaseMedia, + flushOrphanedMedia, }; diff --git a/src/releases.js b/src/releases.js index 40af6666..2fb04114 100644 --- a/src/releases.js +++ b/src/releases.js @@ -1,6 +1,10 @@ 'use strict'; +const inquirer = require('inquirer'); + +const logger = require('./logger')(__filename); const knex = require('./knex'); +const { flushOrphanedMedia } = require('./media'); function curateRelease(release, withMedia = false) { if (!release) { @@ -125,14 +129,52 @@ async function deleteScenes(sceneIds) { await knex('releases') .whereIn('id', sceneIds) .delete(); +} - // TODO: wipe media without associations, clean disk +async function deleteMovies(movieIds) { + await knex('movies') + .whereIn('id', movieIds) + .delete(); +} + +async function flushBatches(batchIds) { + const [sceneIds, movieIds] = await Promise.all([ + knex('releases') + .select('releases.id') + .whereIn('created_batch_id', batchIds) + .pluck('releases.id'), + knex('movies').whereIn('created_batch_id', batchIds) + .select('movies.id') + .whereIn('created_batch_id', batchIds) + .pluck('movies.id'), + ]); + + const confirmed = await inquirer.prompt([{ + type: 'confirm', + name: 'flushBatches', + message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies from batches ${batchIds}. Are you sure?`, + default: false, + }]); + + if (!confirmed.flushBatches) { + logger.warn(`Confirmation rejected, not flushing scenes or movies for batches ${batchIds}`); + return; + } + + await Promise.all([ + deleteScenes(sceneIds), + deleteMovies(movieIds), + ]); + + await flushOrphanedMedia(); } module.exports = { curateRelease, fetchRelease, fetchReleases, + flushBatches, searchReleases, deleteScenes, + deleteMovies, }; diff --git a/src/scrapers/traxxx.js b/src/scrapers/traxxx.js new file mode 100644 index 00000000..054edb49 --- /dev/null +++ b/src/scrapers/traxxx.js @@ -0,0 +1,15 @@ +'use strict'; + +async function fetchLatest() { + return [ + { + title: 'Hot chicks arse fucked', + date: new Date(), + }, + ]; +} + + +module.exports = { + fetchLatest, +};