DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added memory profiling.

This commit is contained in:
DebaucheryLibrarian 2021-10-17 23:56:49 +02:00
parent c1aea78496
commit 972b15e72d
4 changed files with 19758 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ config/*
!config/default.js !config/default.js
assets/js/config/ assets/js/config/
!assets/js/config/default.js !assets/js/config/default.js
*.heapprofile

19692
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -108,6 +108,7 @@
"html-entities": "^2.3.2", "html-entities": "^2.3.2",
"iconv-lite": "^0.5.1", "iconv-lite": "^0.5.1",
"inquirer": "^7.3.3", "inquirer": "^7.3.3",
"inspector-api": "^1.4.2",
"jsdom": "^16.3.0", "jsdom": "^16.3.0",
"knex": "^0.21.13", "knex": "^0.21.13",
"knex-migrate": "^1.7.4", "knex-migrate": "^1.7.4",

238
src/app.js
View File

@ -2,11 +2,15 @@
const util = require('util'); const util = require('util');
const log = require('why-is-node-running'); const log = require('why-is-node-running');
const Inspector = require('inspector-api');
const fs = require('fs').promises;
const dayjs = require('dayjs');
const argv = require('./argv'); const argv = require('./argv');
const initServer = require('./web/server'); const initServer = require('./web/server');
const http = require('./utils/http'); const http = require('./utils/http');
const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const fetchUpdates = require('./updates'); const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep'); const { fetchScenes, fetchMovies } = require('./deep');
@ -17,6 +21,8 @@ const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = r
const { flushOrphanedMedia } = require('./media'); const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries'); const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
function logActive() { function logActive() {
setTimeout(() => { setTimeout(() => {
log(); log();
@ -24,107 +30,141 @@ function logActive() {
}, typeof argv.logActive === 'number' ? argv.logActive : 60000); }, typeof argv.logActive === 'number' ? argv.logActive : 60000);
} }
async function stopMemorySample() {
const profile = await inspector.heap.stopSampling();
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
await inspector.heap.disable();
fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap sample to ${filepath}`);
}
async function init() { async function init() {
if (argv.logActive) { try {
logActive(); if (argv.memory) {
await inspector.heap.enable();
await inspector.heap.startSampling();
logger.info('Started heap sampling');
}
if (argv.logActive) {
logActive();
}
if (argv.server) {
await initServer();
return;
}
if (argv.updateSearch) {
await Promise.all([
updateSceneSearch(),
updateMovieSearch(),
]);
}
if (argv.interpolateProfiles) {
await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null);
}
if (argv.flushActors) {
await flushActors(argv.flushActors);
}
if (argv.flushProfiles) {
await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
}
if (argv.flushNetworks || argv.flushChannels) {
await flushEntities(argv.flushNetworks, argv.flushChannels);
}
if (argv.flushBatches) {
await flushBatches(argv.flushBatches);
}
if (argv.flushScenes) {
await flushScenes();
}
if (argv.flushMovies) {
await flushMovies();
}
if (argv.deleteActors) {
await deleteActors(argv.deleteActors);
}
if (argv.deleteScenes) {
await deleteScenes(argv.deleteScenes);
}
if (argv.deleteMovies) {
await deleteMovies(argv.deleteMovies);
}
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
if (argv.request) {
const res = await http.get(argv.request);
console.log(res.status, res.body);
}
const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
const deepScenes = argv.deep
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
if (argv.report) {
console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
}
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
await associateMovieScenes(storedMovies, storedScenes);
}
if (argv.memory) {
await stopMemorySample();
}
knex.destroy();
} catch (error) {
logger.error(error);
if (argv.memory) {
await stopMemorySample();
}
knex.destroy();
throw error;
} }
if (argv.server) {
await initServer();
return;
}
if (argv.updateSearch) {
await Promise.all([
updateSceneSearch(),
updateMovieSearch(),
]);
}
if (argv.interpolateProfiles) {
await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null);
}
if (argv.flushActors) {
await flushActors(argv.flushActors);
}
if (argv.flushProfiles) {
await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
}
if (argv.flushNetworks || argv.flushChannels) {
await flushEntities(argv.flushNetworks, argv.flushChannels);
}
if (argv.flushBatches) {
await flushBatches(argv.flushBatches);
}
if (argv.flushScenes) {
await flushScenes();
}
if (argv.flushMovies) {
await flushMovies();
}
if (argv.deleteActors) {
await deleteActors(argv.deleteActors);
}
if (argv.deleteScenes) {
await deleteScenes(argv.deleteScenes);
}
if (argv.deleteMovies) {
await deleteMovies(argv.deleteMovies);
}
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
if (argv.request) {
const res = await http.get(argv.request);
console.log(res.status, res.body);
}
const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
const deepScenes = argv.deep
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
if (argv.report) {
console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
}
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
await associateMovieScenes(storedMovies, storedScenes);
}
knex.destroy();
} }
module.exports = init; module.exports = init;