Added orphaned media flush and batch release flush.

assets/components/releases/release.vue

@@ -193,7 +193,7 @@
 					:to="`/added/${formatDate(release.createdAt, 'YYYY/MM/DD')}`"
 					:title="`Added on ${formatDate(release.createdAt, 'MMMM D, YYYY HH:mm')}`"
 					class="link added"
-				>{{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link>
+				>{{ release.createdBatchId }}: {{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link>
 			</div>
 		</div>
 	</div>

assets/js/fragments.js

@@ -185,6 +185,7 @@ const releaseFields = `
 	comment
     createdAt
     url
+	createdBatchId
     ${releaseActorsFragment}
     ${releaseTagsFragment}
     ${releasePosterFragment}
@@ -238,6 +239,7 @@ const releaseFragment = `
     createdAt
     shootId
 	productionDate
+	createdBatchId
 	productionLocation
 	productionCity
 	productionState

migrations/20190325001339_releases.js

@@ -296,7 +296,8 @@ exports.up = knex => Promise.resolve()
 
 		table.integer('batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.datetime('updated_at')
 			.defaultTo(knex.fn.now());
@@ -310,7 +311,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('actor_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors');
+			.inTable('actors')
+			.onDelete('cascade');
 
 		table.integer('entity_id', 12)
 			.references('id')
@@ -509,7 +511,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('actor_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors');
+			.inTable('actors')
+			.onDelete('cascade');
 
 		table.text('body_slug', 20)
 			.references('slug')
@@ -528,7 +531,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('actor_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors');
+			.inTable('actors')
+			.onDelete('cascade');
 
 		table.text('body_slug', 20)
 			.references('slug')
@@ -545,7 +549,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('profile_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors_profiles');
+			.inTable('actors_profiles')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -558,7 +563,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('actor_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors');
+			.inTable('actors')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -645,11 +651,13 @@ exports.up = knex => Promise.resolve()
 
 		table.integer('created_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.integer('updated_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.datetime('created_at')
 			.defaultTo(knex.fn.now());
@@ -664,7 +672,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('actor_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('actors');
+			.inTable('actors')
+			.onDelete('cascade');
 
 		table.unique(['release_id', 'actor_id']);
 
@@ -809,11 +818,13 @@ exports.up = knex => Promise.resolve()
 
 		table.integer('created_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.integer('updated_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.datetime('created_at')
 			.defaultTo(knex.fn.now());
@@ -838,7 +849,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('movie_id', 16)
 			.notNullable()
 			.references('id')
-			.inTable('movies');
+			.inTable('movies')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -852,7 +864,8 @@ exports.up = knex => Promise.resolve()
 			.unique()
 			.notNullable()
 			.references('id')
-			.inTable('movies');
+			.inTable('movies')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -865,7 +878,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('release_id', 12)
 			.references('id')
 			.inTable('releases')
-			.notNullable();
+			.notNullable()
+			.onDelete('cascade');
 
 		table.integer('clip', 6);
 
@@ -879,11 +893,13 @@ exports.up = knex => Promise.resolve()
 
 		table.integer('created_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.integer('updated_batch_id', 12)
 			.references('id')
-			.inTable('batches');
+			.inTable('batches')
+			.onDelete('cascade');
 
 		table.datetime('created_at')
 			.defaultTo(knex.fn.now());
@@ -892,7 +908,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('clip_id', 16)
 			.notNullable()
 			.references('id')
-			.inTable('clips');
+			.inTable('clips')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -905,7 +922,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('clip_id', 16)
 			.notNullable()
 			.references('id')
-			.inTable('clips');
+			.inTable('clips')
+			.onDelete('cascade');
 
 		table.text('media_id', 21)
 			.notNullable()
@@ -918,7 +936,8 @@ exports.up = knex => Promise.resolve()
 		table.integer('tag_id', 12)
 			.notNullable()
 			.references('id')
-			.inTable('tags');
+			.inTable('tags')
+			.onDelete('cascade');
 
 		table.integer('clip_id', 16)
 			.notNullable()

src/app.js

@@ -11,7 +11,8 @@ const { fetchScenes, fetchMovies } = require('./deep');
 const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases');
 const { scrapeActors } = require('./actors');
 const { flushEntities } = require('./entities');
-const { deleteScenes } = require('./releases');
+const { deleteScenes, deleteMovies, flushBatches } = require('./releases');
+const { flushOrphanedMedia } = require('./media');
 const getFileEntries = require('./utils/file-entries');
 
 async function init() {
@@ -28,10 +29,26 @@ async function init() {
 		await flushEntities(argv.flushNetworks, argv.flushChannels);
 	}
 
+	if (argv.flushBatches) {
+		await flushBatches(argv.flushBatches);
+	}
+
+	if (argv.deleteScenes) {
+		await deleteScenes(argv.deleteScenes);
+	}
+
+	if (argv.deleteMovies) {
+		await deleteMovies(argv.deleteMovies);
+	}
+
 	if (argv.delete) {
 		await deleteScenes(argv.delete);
 	}
 
+	if (argv.flushOrphanedMedia) {
+		await flushOrphanedMedia();
+	}
+
 	const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
 	const actorNames = (argv.actors || []).concat(actorsFromFile || []);
 

src/argv.js

@@ -238,6 +238,11 @@ const { argv } = yargs
 		type: 'boolean',
 		default: false,
 	})
+	.option('flush-orphaned-media', {
+		describe: 'Remove all orphaned media items from database and disk.',
+		type: 'array',
+		alias: 'flush-media',
+	})
 	.option('flush-channels', {
 		describe: 'Delete all scenes and movies from channels.',
 		type: 'array',
@@ -248,10 +253,20 @@ const { argv } = yargs
 		type: 'array',
 		alias: 'flush-network',
 	})
-	.option('delete', {
+	.option('flush-batches', {
+		describe: 'Delete all scenes and movies from batch by ID.',
+		type: 'array',
+		alias: 'flush-batch',
+	})
+	.option('delete-scenes', {
 		describe: 'Remove scenes by ID.',
 		type: 'array',
-		alias: 'remove',
+		alias: ['delete-scene', 'delete', 'remove', 'remove-scenes', 'remove-scene'],
+	})
+	.option('delete-movies', {
+		describe: 'Remove movies by ID.',
+		type: 'array',
+		alias: ['delete-movie', 'remove-movies', 'remove-movies'],
 	})
 	.coerce('after', interpretAfter)
 	.coerce('actors-update', interpretAfter);

src/entities.js

@@ -6,7 +6,8 @@ const inquirer = require('inquirer');
 const logger = require('./logger')(__filename);
 const argv = require('./argv');
 const knex = require('./knex');
-const { deleteScenes } = require('./releases');
+const { deleteScenes, deleteMovies } = require('./releases');
+const { flushOrphanedMedia } = require('./media');
 
 function curateEntity(entity, includeParameters = false) {
 	if (!entity) {
@@ -235,7 +236,16 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
 		.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
 		.pluck('releases.id');
 
-	if (sceneIds.length === 0) {
+	const movieIds = await entityQuery
+		.clone()
+		.select('movies.id')
+		.distinct('movies.id')
+		.whereNotNull('movies.id')
+		.from('selected_entities')
+		.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
+		.pluck('movies.id');
+
+	if (sceneIds.length === 0 && movieIds.length === 0) {
 		logger.info(`No scenes or movies found to remove for ${entitySlugs}`);
 		return;
 	}
@@ -243,16 +253,21 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
 	const confirmed = await inquirer.prompt([{
 		type: 'confirm',
 		name: 'flushEntities',
-		message: `You are about to remove ${sceneIds.length} scenes for ${entitySlugs}. Are you sure?`,
+		message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies for ${entitySlugs}. Are you sure?`,
 		default: false,
 	}]);
 
 	if (!confirmed.flushEntities) {
-		logger.warn(`Confirmation rejected, not flushing scenes for: ${entitySlugs}`);
+		logger.warn(`Confirmation rejected, not flushing scenes or movies for: ${entitySlugs}`);
 		return;
 	}
 
-	await deleteScenes(sceneIds);
+	await Promise.all([
+		deleteScenes(sceneIds),
+		deleteMovies(movieIds),
+	]);
+
+	await flushOrphanedMedia();
 }
 
 module.exports = {

src/media.js

@@ -747,7 +747,49 @@ async function associateAvatars(profiles) {
 	return profilesWithAvatarIds;
 }
 
+async function flushOrphanedMedia() {
+	const orphanedMedia = await knex('media')
+		.where('is_sfw', false)
+		.whereNotExists(
+			knex
+				.from(
+					knex('tags_posters')
+						.select('media_id')
+						.unionAll(
+							knex('tags_photos').select('media_id'),
+							knex('releases_posters').select('media_id'),
+							knex('releases_photos').select('media_id'),
+							knex('releases_trailers').select('media_id'),
+							knex('releases_teasers').select('media_id'),
+							knex('movies_covers').select('media_id'),
+							knex('movies_trailers').select('media_id'),
+							knex('actors_avatars').select('media_id'),
+							knex('actors_photos').select('media_id'),
+							knex('clips_photos').select('media_id'),
+							knex('clips_posters').select('media_id'),
+						)
+						.as('associations'),
+				)
+				.whereRaw('associations.media_id = media.id'),
+		)
+		.returning(['media.path', 'media.thumbnail', 'media.lazy'])
+		.delete();
+
+	await Promise.all(orphanedMedia.map(media => Promise.all([
+		fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
+		fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
+		fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
+	])));
+
+	logger.info(`Removed ${orphanedMedia.length} media files from database and disk`);
+
+	await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
+
+	logger.info('Removed temporary media directory');
+}
+
 module.exports = {
 	associateAvatars,
 	associateReleaseMedia,
+	flushOrphanedMedia,
 };

src/releases.js

@@ -1,6 +1,10 @@
 'use strict';
 
+const inquirer = require('inquirer');
+
+const logger = require('./logger')(__filename);
 const knex = require('./knex');
+const { flushOrphanedMedia } = require('./media');
 
 function curateRelease(release, withMedia = false) {
 	if (!release) {
@@ -125,14 +129,52 @@ async function deleteScenes(sceneIds) {
 	await knex('releases')
 		.whereIn('id', sceneIds)
 		.delete();
+}
 
-	// TODO: wipe media without associations, clean disk
+async function deleteMovies(movieIds) {
+	await knex('movies')
+		.whereIn('id', movieIds)
+		.delete();
+}
+
+async function flushBatches(batchIds) {
+	const [sceneIds, movieIds] = await Promise.all([
+		knex('releases')
+			.select('releases.id')
+			.whereIn('created_batch_id', batchIds)
+			.pluck('releases.id'),
+		knex('movies').whereIn('created_batch_id', batchIds)
+			.select('movies.id')
+			.whereIn('created_batch_id', batchIds)
+			.pluck('movies.id'),
+	]);
+
+	const confirmed = await inquirer.prompt([{
+		type: 'confirm',
+		name: 'flushBatches',
+		message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies from batches ${batchIds}. Are you sure?`,
+		default: false,
+	}]);
+
+	if (!confirmed.flushBatches) {
+		logger.warn(`Confirmation rejected, not flushing scenes or movies for batches ${batchIds}`);
+		return;
+	}
+
+	await Promise.all([
+		deleteScenes(sceneIds),
+		deleteMovies(movieIds),
+	]);
+
+	await flushOrphanedMedia();
 }
 
 module.exports = {
 	curateRelease,
 	fetchRelease,
 	fetchReleases,
+	flushBatches,
 	searchReleases,
 	deleteScenes,
+	deleteMovies,
 };

src/scrapers/traxxx.js

@@ -0,0 +1,15 @@
+'use strict';
+
+async function fetchLatest() {
+	return [
+		{
+			title: 'Hot chicks arse fucked',
+			date: new Date(),
+		},
+	];
+}
+
+
+module.exports = {
+	fetchLatest,
+};