Added memory profiling.

.gitignore

@@ -8,3 +8,4 @@ config/*
 !config/default.js
 assets/js/config/
 !assets/js/config/default.js
+*.heapprofile

package.json

@@ -108,6 +108,7 @@
         "html-entities": "^2.3.2",
         "iconv-lite": "^0.5.1",
         "inquirer": "^7.3.3",
+        "inspector-api": "^1.4.2",
         "jsdom": "^16.3.0",
         "knex": "^0.21.13",
         "knex-migrate": "^1.7.4",

src/app.js

@@ -2,11 +2,15 @@
 
 const util = require('util');
 const log = require('why-is-node-running');
+const Inspector = require('inspector-api');
+const fs = require('fs').promises;
+const dayjs = require('dayjs');
 
 const argv = require('./argv');
 const initServer = require('./web/server');
 const http = require('./utils/http');
 
+const logger = require('./logger')(__filename);
 const knex = require('./knex');
 const fetchUpdates = require('./updates');
 const { fetchScenes, fetchMovies } = require('./deep');
@@ -17,6 +21,8 @@ const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = r
 const { flushOrphanedMedia } = require('./media');
 const getFileEntries = require('./utils/file-entries');
 
+const inspector = new Inspector();
+
 function logActive() {
 	setTimeout(() => {
 		log();
@@ -24,107 +30,141 @@ function logActive() {
 	}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
 }
 
+async function stopMemorySample() {
+	const profile = await inspector.heap.stopSampling();
+	const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
+
+	await inspector.heap.disable();
+
+	fs.writeFile(filepath, JSON.stringify(profile));
+
+	logger.info(`Saved heap sample to ${filepath}`);
+}
+
 async function init() {
-	if (argv.logActive) {
+	try {
-		logActive();
+		if (argv.memory) {
+			await inspector.heap.enable();
+			await inspector.heap.startSampling();
+
+			logger.info('Started heap sampling');
+		}
+
+		if (argv.logActive) {
+			logActive();
+		}
+
+		if (argv.server) {
+			await initServer();
+			return;
+		}
+
+		if (argv.updateSearch) {
+			await Promise.all([
+				updateSceneSearch(),
+				updateMovieSearch(),
+			]);
+		}
+
+		if (argv.interpolateProfiles) {
+			await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null);
+		}
+
+		if (argv.flushActors) {
+			await flushActors(argv.flushActors);
+		}
+
+		if (argv.flushProfiles) {
+			await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
+		}
+
+		if (argv.flushNetworks || argv.flushChannels) {
+			await flushEntities(argv.flushNetworks, argv.flushChannels);
+		}
+
+		if (argv.flushBatches) {
+			await flushBatches(argv.flushBatches);
+		}
+
+		if (argv.flushScenes) {
+			await flushScenes();
+		}
+
+		if (argv.flushMovies) {
+			await flushMovies();
+		}
+
+		if (argv.deleteActors) {
+			await deleteActors(argv.deleteActors);
+		}
+
+		if (argv.deleteScenes) {
+			await deleteScenes(argv.deleteScenes);
+		}
+
+		if (argv.deleteMovies) {
+			await deleteMovies(argv.deleteMovies);
+		}
+
+		if (argv.flushOrphanedMedia) {
+			await flushOrphanedMedia();
+		}
+
+		if (argv.request) {
+			const res = await http.get(argv.request);
+
+			console.log(res.status, res.body);
+		}
+
+		const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
+		const actorNames = (argv.actors || []).concat(actorsFromFile || []);
+
+		const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
+		const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
+
+		const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
+
+		const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
+		const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
+
+		const deepScenes = argv.deep
+			? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
+			: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
+
+		const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
+		const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
+
+		const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
+		const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
+
+		if (argv.report) {
+			console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
+			console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
+		}
+
+		if (argv.save) {
+			const storedMovies = await storeMovies(deepMovies);
+			const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
+
+			await associateMovieScenes(storedMovies, storedScenes);
+		}
+
+		if (argv.memory) {
+			await stopMemorySample();
+		}
+
+		knex.destroy();
+	} catch (error) {
+		logger.error(error);
+
+		if (argv.memory) {
+			await stopMemorySample();
+		}
+
+		knex.destroy();
+
+		throw error;
 	}
-
-	if (argv.server) {
-		await initServer();
-		return;
-	}
-
-	if (argv.updateSearch) {
-		await Promise.all([
-			updateSceneSearch(),
-			updateMovieSearch(),
-		]);
-	}
-
-	if (argv.interpolateProfiles) {
-		await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null);
-	}
-
-	if (argv.flushActors) {
-		await flushActors(argv.flushActors);
-	}
-
-	if (argv.flushProfiles) {
-		await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
-	}
-
-	if (argv.flushNetworks || argv.flushChannels) {
-		await flushEntities(argv.flushNetworks, argv.flushChannels);
-	}
-
-	if (argv.flushBatches) {
-		await flushBatches(argv.flushBatches);
-	}
-
-	if (argv.flushScenes) {
-		await flushScenes();
-	}
-
-	if (argv.flushMovies) {
-		await flushMovies();
-	}
-
-	if (argv.deleteActors) {
-		await deleteActors(argv.deleteActors);
-	}
-
-	if (argv.deleteScenes) {
-		await deleteScenes(argv.deleteScenes);
-	}
-
-	if (argv.deleteMovies) {
-		await deleteMovies(argv.deleteMovies);
-	}
-
-	if (argv.flushOrphanedMedia) {
-		await flushOrphanedMedia();
-	}
-
-	if (argv.request) {
-		const res = await http.get(argv.request);
-
-		console.log(res.status, res.body);
-	}
-
-	const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
-	const actorNames = (argv.actors || []).concat(actorsFromFile || []);
-
-	const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
-	const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.scenes).flat().filter(Boolean);
-
-	const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
-
-	const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
-	const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
-
-	const deepScenes = argv.deep
-		? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
-		: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
-
-	const sceneMovies = deepScenes ? deepScenes.filter(scene => scene.movie).map(scene => ({ ...scene.movie, entity: scene.entity })) : [];
-	const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
-
-	const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
-	const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
-
-	if (argv.report) {
-		console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
-		console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
-	}
-
-	if (argv.save) {
-		const storedMovies = await storeMovies(deepMovies);
-		const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
-
-		await associateMovieScenes(storedMovies, storedScenes);
-	}
-
-	knex.destroy();
 }
 
 module.exports = init;