traxxx/src/app.js

'use strict';

const startTime = new Date();

const config = require('config');
const util = require('util');
const unprint = require('unprint');
const log = require('why-is-node-running');
const Inspector = require('inspector-api');
const fs = require('fs').promises;
const { format, intervalToDuration } = require('date-fns');

const argv = require('./argv');
const initServer = require('./web/server');
const http = require('./utils/http');

const logger = require('./logger')(__filename);
const knex = require('./knex');
const redis = require('./redis');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases');
const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries');

const inspector = new Inspector();
let done = false;

unprint.options({
	timeout: argv.requestTimeout,
	headers: {
		'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
	},
	limits: {
		...config.limits,
		default: {
			interval: argv.interval || config.limits.default.interval,
			concurrency: argv.concurrency || config.limits.default.concurrency,
		},
	},
	proxy: config.proxy,
});

unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));

function logActive() {
	setTimeout(() => {
		log();

		if (!done) {
			logActive();
		}
	}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}

async function snapshotMemory(trigger) {
	const profile = await inspector.heap.takeSnapshot();
	const filepath = `traxxx_snapshot_${trigger}M_${format(new Date(), 'yyyy-MM-dd_HH-mm-ss')}.heapsnapshot`;

	logger.info(`Starting heap snapshot, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);

	await inspector.heap.disable();
	await fs.writeFile(filepath, JSON.stringify(profile));

	logger.info(`Saved heap snapshot to ${filepath}`);
}

async function stopMemorySample(snapshotTriggers) {
	const usage = process.memoryUsage.rss() / 1000000;

	const profile = await inspector.heap.stopSampling();
	const filepath = `traxxx_sample_${format(new Date(), 'yyyy-MM-dd_HH-mm-ss')}.heapprofile`;

	await inspector.heap.disable();
	await fs.writeFile(filepath, JSON.stringify(profile));

	logger.info(`Saved heap sample to ${filepath}`);

	if (usage > snapshotTriggers[0]) {
		await snapshotMemory(snapshotTriggers[0]);
		return snapshotTriggers.slice(1);
	}

	return snapshotTriggers;
}

async function startMemorySample(snapshotTriggers = []) {
	await inspector.heap.enable();
	await inspector.heap.startSampling();

	const usage = process.memoryUsage.rss() / 1000000;

	logger.info(`Start heap sampling, memory usage: ${usage} MB`);

	setTimeout(async () => {
		const newSnapshotTriggers = await stopMemorySample(snapshotTriggers);

		if (!done) {
			await startMemorySample(newSnapshotTriggers);
		}
	}, config.memorySampling.sampleDuration);
}

async function init() {
	try {
		if (argv.server) {
			await initServer();
			return;
		}

		if (argv.sampleMemory) {
			await startMemorySample(config.memorySampling.snapshotIntervals);
		}

		if (argv.logActive) {
			logActive();
		}

		if (argv.updateSearch) {
			await Promise.all([
				updateSceneSearch(),
				updateMovieSearch(),
			]);
		}

		if (argv.interpolateProfiles) {
			await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null, true);
		}

		if (argv.flushActors) {
			await flushActors(argv.flushActors);
		}

		if (argv.flushProfiles) {
			await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
		}

		if (argv.flushNetworks || argv.flushChannels) {
			await flushEntities(argv.flushNetworks, argv.flushChannels);
		}

		if (argv.flushBatches) {
			await flushBatches(argv.flushBatches);
		}

		if (argv.flushScenes) {
			await flushScenes();
		}

		if (argv.flushMovies) {
			await flushMovies();
		}

		if (argv.deleteActors) {
			await deleteActors(argv.deleteActors);
		}

		if (argv.deleteScenes) {
			await deleteScenes(argv.deleteScenes);
		}

		if (argv.deleteMovies) {
			await deleteMovies(argv.deleteMovies);
		}

		if (argv.flushOrphanedMedia) {
			await flushOrphanedMedia();
		}

		if (argv.request) {
			const res = await http[argv.requestMethod](argv.request);

			console.log(res.status, res.body);
		}

		const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
		const actorNames = (argv.actors || []).concat(actorsFromFile || []);

		const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
		const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean);

		const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();

		const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
		const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);

		const deepScenes = argv.deep
			? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
			: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];

		const storedScenes = argv.save ? await storeScenes(deepScenes) : [];

		const moviesFromFile = argv.moviesFile && await getFileEntries(argv.moviesFile);
		const movieUrls = (argv.movie || []).concat(moviesFromFile || []);

		const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
		const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;

		const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
		const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;

		if (argv.report) {
			console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
			console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
		}

		if (argv.save) {
			const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
			const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);

			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
		}

		logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() }))
			.filter(([, value]) => value > 0)
			.map(([key, value]) => `${value} ${key}`)
			.join(', ')}`);
	} catch (error) {
		console.trace(error);
		logger.error(error);
	}

	await http.destroyBypassSessions();
	await http.destroyBrowserSessions();

	knex.destroy();
	redis.disconnect();

	done = true;
}

module.exports = init;