DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Merge branch 'master' into experimental

Browse Source
This commit is contained in:
DebaucheryLibrarian
2022-03-30 23:00:29 +02:00
parent ccb99e278c a46061e247
commit 33a327a04b
1105 changed files with 6700 additions and 1561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/app.js
View File

@@ -1,5 +1,6 @@
'use strict';
const config = require('config');
const util = require('util');
// const log = require('why-is-node-running');
const Inspector = require('inspector-api');
@@ -24,33 +25,64 @@ const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
let done = false;
function logActive() {
console.log('log active!');
setTimeout(() => {
// log();
logActive();
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}
/*
function monitorMemory() {
logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
function logActive() {
setTimeout(() => {
log();
if (!done) {
setTimeout(() => monitorMemory(), 10000);
}
if (!done) {
logActive();
}
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}
*/
async function stopMemorySample() {
async function snapshotMemory(trigger) {
const profile = await inspector.heap.takeSnapshot();
const filepath = `traxxx_snapshot_${trigger}M_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapsnapshot`;
logger.info(`Starting heap snapshot, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap snapshot to ${filepath}`);
}
async function stopMemorySample(snapshotTriggers) {
const usage = process.memoryUsage.rss() / 1000000;
const profile = await inspector.heap.stopSampling();
const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
const filepath = `traxxx_sample_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
logger.info(`Saved heap sample to ${filepath}`);
if (usage > snapshotTriggers[0]) {
await snapshotMemory(snapshotTriggers[0]);
return snapshotTriggers.slice(1);
}
return snapshotTriggers;
}
async function startMemorySample(snapshotTriggers = []) {
await inspector.heap.enable();
await inspector.heap.startSampling();
const usage = process.memoryUsage.rss() / 1000000;
logger.info(`Start heap sampling, memory usage: ${usage} MB`);
setTimeout(async () => {
const newSnapshotTriggers = await stopMemorySample(snapshotTriggers);
if (!done) {
await startMemorySample(newSnapshotTriggers);
}
}, config.memorySampling.sampleDuration);
}
async function startMemorySample() {
@@ -72,19 +104,19 @@ async function startMemorySample() {
async function init() {
try {
if (argv.memory) {
await startMemorySample();
}
if (argv.logActive) {
logActive();
}
if (argv.server) {
await initServer();
return;
}
if (argv.sampleMemory) {
await startMemorySample(config.memorySampling.snapshotIntervals);
}
if (argv.logActive) {
// logActive();
}
if (argv.updateSearch) {
await Promise.all([
updateSceneSearch(),
@@ -157,8 +189,13 @@ async function init() {
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const storedScenes = argv.save ? await storeScenes(deepScenes) : [];
const moviesFromFile = argv.moviesFile && await getFileEntries(argv.moviesFile);
const movieUrls = (argv.movie || []).concat(moviesFromFile || []);
const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
@@ -169,10 +206,10 @@ async function init() {
}
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
await associateMovieScenes(storedMovies, storedScenes);
await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
}
} catch (error) {
logger.error(error);