Added orphaned media flush and batch release flush.

This commit is contained in:
DebaucheryLibrarian 2020-10-25 00:52:40 +02:00
parent ef852f0191
commit 0bd7fca876
9 changed files with 196 additions and 29 deletions

View File

@ -193,7 +193,7 @@
:to="`/added/${formatDate(release.createdAt, 'YYYY/MM/DD')}`" :to="`/added/${formatDate(release.createdAt, 'YYYY/MM/DD')}`"
:title="`Added on ${formatDate(release.createdAt, 'MMMM D, YYYY HH:mm')}`" :title="`Added on ${formatDate(release.createdAt, 'MMMM D, YYYY HH:mm')}`"
class="link added" class="link added"
>{{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link> >{{ release.createdBatchId }}: {{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link>
</div> </div>
</div> </div>
</div> </div>

View File

@ -185,6 +185,7 @@ const releaseFields = `
comment comment
createdAt createdAt
url url
createdBatchId
${releaseActorsFragment} ${releaseActorsFragment}
${releaseTagsFragment} ${releaseTagsFragment}
${releasePosterFragment} ${releasePosterFragment}
@ -238,6 +239,7 @@ const releaseFragment = `
createdAt createdAt
shootId shootId
productionDate productionDate
createdBatchId
productionLocation productionLocation
productionCity productionCity
productionState productionState

View File

@ -296,7 +296,8 @@ exports.up = knex => Promise.resolve()
table.integer('batch_id', 12) table.integer('batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.datetime('updated_at') table.datetime('updated_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
@ -310,7 +311,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12) table.integer('actor_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors')
.onDelete('cascade');
table.integer('entity_id', 12) table.integer('entity_id', 12)
.references('id') .references('id')
@ -509,7 +511,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12) table.integer('actor_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors')
.onDelete('cascade');
table.text('body_slug', 20) table.text('body_slug', 20)
.references('slug') .references('slug')
@ -528,7 +531,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12) table.integer('actor_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors')
.onDelete('cascade');
table.text('body_slug', 20) table.text('body_slug', 20)
.references('slug') .references('slug')
@ -545,7 +549,8 @@ exports.up = knex => Promise.resolve()
table.integer('profile_id', 12) table.integer('profile_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors_profiles'); .inTable('actors_profiles')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -558,7 +563,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12) table.integer('actor_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -645,11 +651,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12) table.integer('created_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12) table.integer('updated_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
@ -664,7 +672,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12) table.integer('actor_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('actors'); .inTable('actors')
.onDelete('cascade');
table.unique(['release_id', 'actor_id']); table.unique(['release_id', 'actor_id']);
@ -809,11 +818,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12) table.integer('created_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12) table.integer('updated_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
@ -838,7 +849,8 @@ exports.up = knex => Promise.resolve()
table.integer('movie_id', 16) table.integer('movie_id', 16)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('movies'); .inTable('movies')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -852,7 +864,8 @@ exports.up = knex => Promise.resolve()
.unique() .unique()
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('movies'); .inTable('movies')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -865,7 +878,8 @@ exports.up = knex => Promise.resolve()
table.integer('release_id', 12) table.integer('release_id', 12)
.references('id') .references('id')
.inTable('releases') .inTable('releases')
.notNullable(); .notNullable()
.onDelete('cascade');
table.integer('clip', 6); table.integer('clip', 6);
@ -879,11 +893,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12) table.integer('created_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12) table.integer('updated_batch_id', 12)
.references('id') .references('id')
.inTable('batches'); .inTable('batches')
.onDelete('cascade');
table.datetime('created_at') table.datetime('created_at')
.defaultTo(knex.fn.now()); .defaultTo(knex.fn.now());
@ -892,7 +908,8 @@ exports.up = knex => Promise.resolve()
table.integer('clip_id', 16) table.integer('clip_id', 16)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('clips'); .inTable('clips')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -905,7 +922,8 @@ exports.up = knex => Promise.resolve()
table.integer('clip_id', 16) table.integer('clip_id', 16)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('clips'); .inTable('clips')
.onDelete('cascade');
table.text('media_id', 21) table.text('media_id', 21)
.notNullable() .notNullable()
@ -918,7 +936,8 @@ exports.up = knex => Promise.resolve()
table.integer('tag_id', 12) table.integer('tag_id', 12)
.notNullable() .notNullable()
.references('id') .references('id')
.inTable('tags'); .inTable('tags')
.onDelete('cascade');
table.integer('clip_id', 16) table.integer('clip_id', 16)
.notNullable() .notNullable()

View File

@ -11,7 +11,8 @@ const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases'); const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases');
const { scrapeActors } = require('./actors'); const { scrapeActors } = require('./actors');
const { flushEntities } = require('./entities'); const { flushEntities } = require('./entities');
const { deleteScenes } = require('./releases'); const { deleteScenes, deleteMovies, flushBatches } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries'); const getFileEntries = require('./utils/file-entries');
async function init() { async function init() {
@ -28,10 +29,26 @@ async function init() {
await flushEntities(argv.flushNetworks, argv.flushChannels); await flushEntities(argv.flushNetworks, argv.flushChannels);
} }
if (argv.flushBatches) {
await flushBatches(argv.flushBatches);
}
if (argv.deleteScenes) {
await deleteScenes(argv.deleteScenes);
}
if (argv.deleteMovies) {
await deleteMovies(argv.deleteMovies);
}
if (argv.delete) { if (argv.delete) {
await deleteScenes(argv.delete); await deleteScenes(argv.delete);
} }
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile); const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
const actorNames = (argv.actors || []).concat(actorsFromFile || []); const actorNames = (argv.actors || []).concat(actorsFromFile || []);

View File

@ -238,6 +238,11 @@ const { argv } = yargs
type: 'boolean', type: 'boolean',
default: false, default: false,
}) })
.option('flush-orphaned-media', {
describe: 'Remove all orphaned media items from database and disk.',
type: 'array',
alias: 'flush-media',
})
.option('flush-channels', { .option('flush-channels', {
describe: 'Delete all scenes and movies from channels.', describe: 'Delete all scenes and movies from channels.',
type: 'array', type: 'array',
@ -248,10 +253,20 @@ const { argv } = yargs
type: 'array', type: 'array',
alias: 'flush-network', alias: 'flush-network',
}) })
.option('delete', { .option('flush-batches', {
describe: 'Delete all scenes and movies from batch by ID.',
type: 'array',
alias: 'flush-batch',
})
.option('delete-scenes', {
describe: 'Remove scenes by ID.', describe: 'Remove scenes by ID.',
type: 'array', type: 'array',
alias: 'remove', alias: ['delete-scene', 'delete', 'remove', 'remove-scenes', 'remove-scene'],
})
.option('delete-movies', {
describe: 'Remove movies by ID.',
type: 'array',
alias: ['delete-movie', 'remove-movies', 'remove-movies'],
}) })
.coerce('after', interpretAfter) .coerce('after', interpretAfter)
.coerce('actors-update', interpretAfter); .coerce('actors-update', interpretAfter);

View File

@ -6,7 +6,8 @@ const inquirer = require('inquirer');
const logger = require('./logger')(__filename); const logger = require('./logger')(__filename);
const argv = require('./argv'); const argv = require('./argv');
const knex = require('./knex'); const knex = require('./knex');
const { deleteScenes } = require('./releases'); const { deleteScenes, deleteMovies } = require('./releases');
const { flushOrphanedMedia } = require('./media');
function curateEntity(entity, includeParameters = false) { function curateEntity(entity, includeParameters = false) {
if (!entity) { if (!entity) {
@ -235,7 +236,16 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id') .leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
.pluck('releases.id'); .pluck('releases.id');
if (sceneIds.length === 0) { const movieIds = await entityQuery
.clone()
.select('movies.id')
.distinct('movies.id')
.whereNotNull('movies.id')
.from('selected_entities')
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
.pluck('movies.id');
if (sceneIds.length === 0 && movieIds.length === 0) {
logger.info(`No scenes or movies found to remove for ${entitySlugs}`); logger.info(`No scenes or movies found to remove for ${entitySlugs}`);
return; return;
} }
@ -243,16 +253,21 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
const confirmed = await inquirer.prompt([{ const confirmed = await inquirer.prompt([{
type: 'confirm', type: 'confirm',
name: 'flushEntities', name: 'flushEntities',
message: `You are about to remove ${sceneIds.length} scenes for ${entitySlugs}. Are you sure?`, message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies for ${entitySlugs}. Are you sure?`,
default: false, default: false,
}]); }]);
if (!confirmed.flushEntities) { if (!confirmed.flushEntities) {
logger.warn(`Confirmation rejected, not flushing scenes for: ${entitySlugs}`); logger.warn(`Confirmation rejected, not flushing scenes or movies for: ${entitySlugs}`);
return; return;
} }
await deleteScenes(sceneIds); await Promise.all([
deleteScenes(sceneIds),
deleteMovies(movieIds),
]);
await flushOrphanedMedia();
} }
module.exports = { module.exports = {

View File

@ -747,7 +747,49 @@ async function associateAvatars(profiles) {
return profilesWithAvatarIds; return profilesWithAvatarIds;
} }
async function flushOrphanedMedia() {
const orphanedMedia = await knex('media')
.where('is_sfw', false)
.whereNotExists(
knex
.from(
knex('tags_posters')
.select('media_id')
.unionAll(
knex('tags_photos').select('media_id'),
knex('releases_posters').select('media_id'),
knex('releases_photos').select('media_id'),
knex('releases_trailers').select('media_id'),
knex('releases_teasers').select('media_id'),
knex('movies_covers').select('media_id'),
knex('movies_trailers').select('media_id'),
knex('actors_avatars').select('media_id'),
knex('actors_photos').select('media_id'),
knex('clips_photos').select('media_id'),
knex('clips_posters').select('media_id'),
)
.as('associations'),
)
.whereRaw('associations.media_id = media.id'),
)
.returning(['media.path', 'media.thumbnail', 'media.lazy'])
.delete();
await Promise.all(orphanedMedia.map(media => Promise.all([
fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
])));
logger.info(`Removed ${orphanedMedia.length} media files from database and disk`);
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Removed temporary media directory');
}
module.exports = { module.exports = {
associateAvatars, associateAvatars,
associateReleaseMedia, associateReleaseMedia,
flushOrphanedMedia,
}; };

View File

@ -1,6 +1,10 @@
'use strict'; 'use strict';
const inquirer = require('inquirer');
const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const { flushOrphanedMedia } = require('./media');
function curateRelease(release, withMedia = false) { function curateRelease(release, withMedia = false) {
if (!release) { if (!release) {
@ -125,14 +129,52 @@ async function deleteScenes(sceneIds) {
await knex('releases') await knex('releases')
.whereIn('id', sceneIds) .whereIn('id', sceneIds)
.delete(); .delete();
}
// TODO: wipe media without associations, clean disk async function deleteMovies(movieIds) {
await knex('movies')
.whereIn('id', movieIds)
.delete();
}
async function flushBatches(batchIds) {
const [sceneIds, movieIds] = await Promise.all([
knex('releases')
.select('releases.id')
.whereIn('created_batch_id', batchIds)
.pluck('releases.id'),
knex('movies').whereIn('created_batch_id', batchIds)
.select('movies.id')
.whereIn('created_batch_id', batchIds)
.pluck('movies.id'),
]);
const confirmed = await inquirer.prompt([{
type: 'confirm',
name: 'flushBatches',
message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies from batches ${batchIds}. Are you sure?`,
default: false,
}]);
if (!confirmed.flushBatches) {
logger.warn(`Confirmation rejected, not flushing scenes or movies for batches ${batchIds}`);
return;
}
await Promise.all([
deleteScenes(sceneIds),
deleteMovies(movieIds),
]);
await flushOrphanedMedia();
} }
module.exports = { module.exports = {
curateRelease, curateRelease,
fetchRelease, fetchRelease,
fetchReleases, fetchReleases,
flushBatches,
searchReleases, searchReleases,
deleteScenes, deleteScenes,
deleteMovies,
}; };

15
src/scrapers/traxxx.js Normal file
View File

@ -0,0 +1,15 @@
'use strict';
async function fetchLatest() {
return [
{
title: 'Hot chicks arse fucked',
date: new Date(),
},
];
}
module.exports = {
fetchLatest,
};