DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added orphaned media flush and batch release flush.

This commit is contained in:
DebaucheryLibrarian 2020-10-25 00:52:40 +02:00
parent ef852f0191
commit 0bd7fca876
9 changed files with 196 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/components/releases/release.vue
View File

@ -193,7 +193,7 @@
:to="`/added/${formatDate(release.createdAt, 'YYYY/MM/DD')}`"
:title="`Added on ${formatDate(release.createdAt, 'MMMM D, YYYY HH:mm')}`"
class="link added"
>{{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link>
>{{ release.createdBatchId }}: {{ formatDate(release.createdAt, 'MMMM D, YYYY HH:mm') }}</router-link>
</div>
</div>
</div>

2
assets/js/fragments.js
View File

@ -185,6 +185,7 @@ const releaseFields = `
comment
createdAt
url
createdBatchId
${releaseActorsFragment}
${releaseTagsFragment}
${releasePosterFragment}
@ -238,6 +239,7 @@ const releaseFragment = `
createdAt
shootId
productionDate
createdBatchId
productionLocation
productionCity
productionState

57
migrations/20190325001339_releases.js
View File

@ -296,7 +296,8 @@ exports.up = knex => Promise.resolve()
table.integer('batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.datetime('updated_at')
.defaultTo(knex.fn.now());
@ -310,7 +311,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
.inTable('actors')
.onDelete('cascade');
table.integer('entity_id', 12)
.references('id')
@ -509,7 +511,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
.inTable('actors')
.onDelete('cascade');
table.text('body_slug', 20)
.references('slug')
@ -528,7 +531,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
.inTable('actors')
.onDelete('cascade');
table.text('body_slug', 20)
.references('slug')
@ -545,7 +549,8 @@ exports.up = knex => Promise.resolve()
table.integer('profile_id', 12)
.notNullable()
.references('id')
.inTable('actors_profiles');
.inTable('actors_profiles')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -558,7 +563,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
.inTable('actors')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -645,11 +651,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.datetime('created_at')
.defaultTo(knex.fn.now());
@ -664,7 +672,8 @@ exports.up = knex => Promise.resolve()
table.integer('actor_id', 12)
.notNullable()
.references('id')
.inTable('actors');
.inTable('actors')
.onDelete('cascade');
table.unique(['release_id', 'actor_id']);
@ -809,11 +818,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.datetime('created_at')
.defaultTo(knex.fn.now());
@ -838,7 +849,8 @@ exports.up = knex => Promise.resolve()
table.integer('movie_id', 16)
.notNullable()
.references('id')
.inTable('movies');
.inTable('movies')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -852,7 +864,8 @@ exports.up = knex => Promise.resolve()
.unique()
.notNullable()
.references('id')
.inTable('movies');
.inTable('movies')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -865,7 +878,8 @@ exports.up = knex => Promise.resolve()
table.integer('release_id', 12)
.references('id')
.inTable('releases')
.notNullable();
.notNullable()
.onDelete('cascade');
table.integer('clip', 6);
@ -879,11 +893,13 @@ exports.up = knex => Promise.resolve()
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
.inTable('batches')
.onDelete('cascade');
table.datetime('created_at')
.defaultTo(knex.fn.now());
@ -892,7 +908,8 @@ exports.up = knex => Promise.resolve()
table.integer('clip_id', 16)
.notNullable()
.references('id')
.inTable('clips');
.inTable('clips')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -905,7 +922,8 @@ exports.up = knex => Promise.resolve()
table.integer('clip_id', 16)
.notNullable()
.references('id')
.inTable('clips');
.inTable('clips')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
@ -918,7 +936,8 @@ exports.up = knex => Promise.resolve()
table.integer('tag_id', 12)
.notNullable()
.references('id')
.inTable('tags');
.inTable('tags')
.onDelete('cascade');
table.integer('clip_id', 16)
.notNullable()

19
src/app.js
View File

@ -11,7 +11,8 @@ const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases');
const { scrapeActors } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes } = require('./releases');
const { deleteScenes, deleteMovies, flushBatches } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries');
async function init() {
@ -28,10 +29,26 @@ async function init() {
await flushEntities(argv.flushNetworks, argv.flushChannels);
}
if (argv.flushBatches) {
await flushBatches(argv.flushBatches);
}
if (argv.deleteScenes) {
await deleteScenes(argv.deleteScenes);
}
if (argv.deleteMovies) {
await deleteMovies(argv.deleteMovies);
}
if (argv.delete) {
await deleteScenes(argv.delete);
}
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
const actorNames = (argv.actors || []).concat(actorsFromFile || []);

19
src/argv.js
View File

@ -238,6 +238,11 @@ const { argv } = yargs
type: 'boolean',
default: false,
})
.option('flush-orphaned-media', {
describe: 'Remove all orphaned media items from database and disk.',
type: 'array',
alias: 'flush-media',
})
.option('flush-channels', {
describe: 'Delete all scenes and movies from channels.',
type: 'array',
@ -248,10 +253,20 @@ const { argv } = yargs
type: 'array',
alias: 'flush-network',
})
.option('delete', {
.option('flush-batches', {
describe: 'Delete all scenes and movies from batch by ID.',
type: 'array',
alias: 'flush-batch',
})
.option('delete-scenes', {
describe: 'Remove scenes by ID.',
type: 'array',
alias: 'remove',
alias: ['delete-scene', 'delete', 'remove', 'remove-scenes', 'remove-scene'],
})
.option('delete-movies', {
describe: 'Remove movies by ID.',
type: 'array',
alias: ['delete-movie', 'remove-movies', 'remove-movies'],
})
.coerce('after', interpretAfter)
.coerce('actors-update', interpretAfter);

25
src/entities.js
View File

@ -6,7 +6,8 @@ const inquirer = require('inquirer');
const logger = require('./logger')(__filename);
const argv = require('./argv');
const knex = require('./knex');
const { deleteScenes } = require('./releases');
const { deleteScenes, deleteMovies } = require('./releases');
const { flushOrphanedMedia } = require('./media');
function curateEntity(entity, includeParameters = false) {
if (!entity) {
@ -235,7 +236,16 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
.leftJoin('releases', 'releases.entity_id', 'selected_entities.id')
.pluck('releases.id');
if (sceneIds.length === 0) {
const movieIds = await entityQuery
.clone()
.select('movies.id')
.distinct('movies.id')
.whereNotNull('movies.id')
.from('selected_entities')
.leftJoin('movies', 'movies.entity_id', 'selected_entities.id')
.pluck('movies.id');
if (sceneIds.length === 0 && movieIds.length === 0) {
logger.info(`No scenes or movies found to remove for ${entitySlugs}`);
return;
}
@ -243,16 +253,21 @@ async function flushEntities(networkSlugs = [], channelSlugs = []) {
const confirmed = await inquirer.prompt([{
type: 'confirm',
name: 'flushEntities',
message: `You are about to remove ${sceneIds.length} scenes for ${entitySlugs}. Are you sure?`,
message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies for ${entitySlugs}. Are you sure?`,
default: false,
}]);
if (!confirmed.flushEntities) {
logger.warn(`Confirmation rejected, not flushing scenes for: ${entitySlugs}`);
logger.warn(`Confirmation rejected, not flushing scenes or movies for: ${entitySlugs}`);
return;
}
await deleteScenes(sceneIds);
await Promise.all([
deleteScenes(sceneIds),
deleteMovies(movieIds),
]);
await flushOrphanedMedia();
}
module.exports = {

42
src/media.js
View File

@ -747,7 +747,49 @@ async function associateAvatars(profiles) {
return profilesWithAvatarIds;
}
async function flushOrphanedMedia() {
const orphanedMedia = await knex('media')
.where('is_sfw', false)
.whereNotExists(
knex
.from(
knex('tags_posters')
.select('media_id')
.unionAll(
knex('tags_photos').select('media_id'),
knex('releases_posters').select('media_id'),
knex('releases_photos').select('media_id'),
knex('releases_trailers').select('media_id'),
knex('releases_teasers').select('media_id'),
knex('movies_covers').select('media_id'),
knex('movies_trailers').select('media_id'),
knex('actors_avatars').select('media_id'),
knex('actors_photos').select('media_id'),
knex('clips_photos').select('media_id'),
knex('clips_posters').select('media_id'),
)
.as('associations'),
)
.whereRaw('associations.media_id = media.id'),
)
.returning(['media.path', 'media.thumbnail', 'media.lazy'])
.delete();
await Promise.all(orphanedMedia.map(media => Promise.all([
fsPromises.unlink(path.join(config.media.path, media.path)).catch(() => { /* probably file not found */ }),
fsPromises.unlink(path.join(config.media.path, media.thumbnail)).catch(() => { /* probably file not found */ }),
fsPromises.unlink(path.join(config.media.path, media.lazy)).catch(() => { /* probably file not found */ }),
])));
logger.info(`Removed ${orphanedMedia.length} media files from database and disk`);
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Removed temporary media directory');
}
module.exports = {
associateAvatars,
associateReleaseMedia,
flushOrphanedMedia,
};

44
src/releases.js
View File

@ -1,6 +1,10 @@
'use strict';
const inquirer = require('inquirer');
const logger = require('./logger')(__filename);
const knex = require('./knex');
const { flushOrphanedMedia } = require('./media');
function curateRelease(release, withMedia = false) {
if (!release) {
@ -125,14 +129,52 @@ async function deleteScenes(sceneIds) {
await knex('releases')
.whereIn('id', sceneIds)
.delete();
}
// TODO: wipe media without associations, clean disk
async function deleteMovies(movieIds) {
await knex('movies')
.whereIn('id', movieIds)
.delete();
}
async function flushBatches(batchIds) {
const [sceneIds, movieIds] = await Promise.all([
knex('releases')
.select('releases.id')
.whereIn('created_batch_id', batchIds)
.pluck('releases.id'),
knex('movies').whereIn('created_batch_id', batchIds)
.select('movies.id')
.whereIn('created_batch_id', batchIds)
.pluck('movies.id'),
]);
const confirmed = await inquirer.prompt([{
type: 'confirm',
name: 'flushBatches',
message: `You are about to remove ${sceneIds.length} scenes and ${movieIds.length} movies from batches ${batchIds}. Are you sure?`,
default: false,
}]);
if (!confirmed.flushBatches) {
logger.warn(`Confirmation rejected, not flushing scenes or movies for batches ${batchIds}`);
return;
}
await Promise.all([
deleteScenes(sceneIds),
deleteMovies(movieIds),
]);
await flushOrphanedMedia();
}
module.exports = {
curateRelease,
fetchRelease,
fetchReleases,
flushBatches,
searchReleases,
deleteScenes,
deleteMovies,
};

15
src/scrapers/traxxx.js Normal file
View File

@ -0,0 +1,15 @@
'use strict';
async function fetchLatest() {
return [
{
title: 'Hot chicks arse fucked',
date: new Date(),
},
];
}
module.exports = {
fetchLatest,
};