traxxx/src/app.js

219 lines
6.5 KiB
JavaScript
Raw Normal View History

'use strict';
const config = require('config');
2020-05-20 00:23:45 +00:00
const util = require('util');
const unprint = require('unprint');
// const log = require('why-is-node-running');
2021-10-17 21:56:49 +00:00
const Inspector = require('inspector-api');
const fs = require('fs').promises;
const dayjs = require('dayjs');
2020-05-20 00:23:45 +00:00
const argv = require('./argv');
const initServer = require('./web/server');
const http = require('./utils/http');
2021-10-17 21:56:49 +00:00
const logger = require('./logger')(__filename);
const knex = require('./knex');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases');
const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
const { flushOrphanedMedia } = require('./media');
const getFileEntries = require('./utils/file-entries');
2021-10-17 21:56:49 +00:00
const inspector = new Inspector();
2021-11-20 22:59:15 +00:00
let done = false;
2021-10-17 21:56:49 +00:00
unprint.options({
timeout: 5000,
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
},
});
/*
function logActive() {
setTimeout(() => {
log();
if (!done) {
logActive();
}
}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
}
*/
async function snapshotMemory(trigger) {
const profile = await inspector.heap.takeSnapshot();
const filepath = `traxxx_snapshot_${trigger}M_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapsnapshot`;
2021-11-20 22:59:15 +00:00
2021-12-27 16:29:23 +00:00
logger.info(`Starting heap snapshot, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
await inspector.heap.disable();
await fs.writeFile(filepath, JSON.stringify(profile));
2021-12-27 16:29:23 +00:00
logger.info(`Saved heap snapshot to ${filepath}`);
2021-11-20 22:59:15 +00:00
}
2021-12-20 01:28:59 +00:00
async function stopMemorySample(snapshotTriggers) {
const usage = process.memoryUsage.rss() / 1000000;
2021-10-17 21:56:49 +00:00
const profile = await inspector.heap.stopSampling();
2021-12-20 02:29:14 +00:00
const filepath = `traxxx_sample_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
2021-10-17 21:56:49 +00:00
await inspector.heap.disable();
2021-11-20 22:59:15 +00:00
await fs.writeFile(filepath, JSON.stringify(profile));
2021-10-17 21:56:49 +00:00
logger.info(`Saved heap sample to ${filepath}`);
2021-12-20 01:28:59 +00:00
if (usage > snapshotTriggers[0]) {
await snapshotMemory(snapshotTriggers[0]);
2021-12-20 01:28:59 +00:00
return snapshotTriggers.slice(1);
}
2021-12-20 01:28:59 +00:00
return snapshotTriggers;
2021-10-17 21:56:49 +00:00
}
2021-12-20 01:28:59 +00:00
async function startMemorySample(snapshotTriggers = []) {
await inspector.heap.enable();
await inspector.heap.startSampling();
const usage = process.memoryUsage.rss() / 1000000;
logger.info(`Start heap sampling, memory usage: ${usage} MB`);
setTimeout(async () => {
2021-12-20 01:28:59 +00:00
const newSnapshotTriggers = await stopMemorySample(snapshotTriggers);
if (!done) {
2021-12-20 01:28:59 +00:00
await startMemorySample(newSnapshotTriggers);
}
}, config.memorySampling.sampleDuration);
2021-10-17 21:56:49 +00:00
}
async function init() {
2021-10-17 21:56:49 +00:00
try {
if (argv.server) {
await initServer();
return;
2021-10-17 21:56:49 +00:00
}
if (argv.sampleMemory) {
await startMemorySample(config.memorySampling.snapshotIntervals);
2021-10-17 21:56:49 +00:00
}
2021-10-17 21:56:49 +00:00
if (argv.logActive) {
// logActive();
2021-10-17 21:56:49 +00:00
}
2020-12-30 01:23:43 +00:00
2021-10-17 21:56:49 +00:00
if (argv.updateSearch) {
await Promise.all([
updateSceneSearch(),
updateMovieSearch(),
]);
}
2020-12-30 02:19:09 +00:00
2021-10-17 21:56:49 +00:00
if (argv.interpolateProfiles) {
await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null);
}
2021-10-17 21:56:49 +00:00
if (argv.flushActors) {
await flushActors(argv.flushActors);
}
2021-10-17 21:56:49 +00:00
if (argv.flushProfiles) {
await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null);
}
2021-10-17 21:56:49 +00:00
if (argv.flushNetworks || argv.flushChannels) {
await flushEntities(argv.flushNetworks, argv.flushChannels);
}
2021-10-17 21:56:49 +00:00
if (argv.flushBatches) {
await flushBatches(argv.flushBatches);
}
2021-10-17 21:56:49 +00:00
if (argv.flushScenes) {
await flushScenes();
}
2021-10-17 21:56:49 +00:00
if (argv.flushMovies) {
await flushMovies();
}
2021-10-17 21:56:49 +00:00
if (argv.deleteActors) {
await deleteActors(argv.deleteActors);
}
2021-10-17 21:56:49 +00:00
if (argv.deleteScenes) {
await deleteScenes(argv.deleteScenes);
}
2021-10-17 21:56:49 +00:00
if (argv.deleteMovies) {
await deleteMovies(argv.deleteMovies);
}
2021-10-17 21:56:49 +00:00
if (argv.flushOrphanedMedia) {
await flushOrphanedMedia();
}
2021-10-17 21:56:49 +00:00
if (argv.request) {
const res = await http[argv.requestMethod](argv.request);
2021-10-17 21:56:49 +00:00
console.log(res.status, res.body);
}
2021-10-17 21:56:49 +00:00
const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile);
const actorNames = (argv.actors || []).concat(actorsFromFile || []);
2021-10-17 21:56:49 +00:00
const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
2021-11-20 22:59:15 +00:00
const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean);
2021-10-17 21:56:49 +00:00
const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
2021-10-17 21:56:49 +00:00
const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
2021-10-17 21:56:49 +00:00
const deepScenes = argv.deep
? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
2021-12-20 02:29:14 +00:00
const storedScenes = argv.save ? await storeScenes(deepScenes) : [];
const moviesFromFile = argv.moviesFile && await getFileEntries(argv.moviesFile);
const movieUrls = (argv.movie || []).concat(moviesFromFile || []);
const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;
2020-05-20 00:23:45 +00:00
2021-11-20 22:59:15 +00:00
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
2021-10-17 21:56:49 +00:00
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
2021-01-26 23:21:58 +00:00
2021-10-17 21:56:49 +00:00
if (argv.report) {
console.log(util.inspect(deepScenes, { depth: Infinity, colors: true }));
console.log(util.inspect(deepMovies, { depth: Infinity, colors: true }));
}
2021-10-17 21:56:49 +00:00
if (argv.save) {
const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
2021-10-17 21:56:49 +00:00
2021-12-20 02:29:14 +00:00
await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
2021-10-17 21:56:49 +00:00
}
} catch (error) {
console.trace(error);
2021-10-17 21:56:49 +00:00
logger.error(error);
}
2021-11-20 22:59:15 +00:00
await http.destroyBypassSessions();
await http.destroyBrowserSessions();
2021-11-20 22:59:15 +00:00
knex.destroy();
done = true;
}
2019-12-09 04:00:49 +00:00
module.exports = init;