'use strict'; const startTime = new Date(); const config = require('config'); const util = require('util'); const unprint = require('unprint'); const log = require('why-is-node-running'); const Inspector = require('inspector-api'); const fs = require('fs').promises; const { format, intervalToDuration } = require('date-fns'); const argv = require('./argv'); const initServer = require('./web/server'); const http = require('./utils/http'); const logger = require('./logger')(__filename); const knex = require('./knex'); const redis = require('./redis'); const fetchUpdates = require('./updates'); const { fetchScenes, fetchMovies } = require('./deep'); const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases'); const { updateSceneSearch, updateMovieSearch } = require('./update-search'); const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors'); const { flushEntities } = require('./entities'); const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases'); const { flushOrphanedMedia } = require('./media'); const getFileEntries = require('./utils/file-entries'); const inspector = new Inspector(); let done = false; unprint.options({ timeout: argv.requestTimeout, headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', }, limits: { ...config.limits, default: { interval: argv.interval || config.limits.default.interval, concurrency: argv.concurrency || config.limits.default.concurrency, }, }, proxy: config.proxy, }); unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}) ${event.url}`)); unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`)); function logActive() { setTimeout(() => { log(); if (!done) { logActive(); } }, typeof argv.logActive === 'number' ? argv.logActive : 60000); } async function snapshotMemory(trigger) { const profile = await inspector.heap.takeSnapshot(); const filepath = `traxxx_snapshot_${trigger}M_${format(new Date(), 'yyyy-MM-dd_HH-mm-ss')}.heapsnapshot`; logger.info(`Starting heap snapshot, memory usage: ${process.memoryUsage.rss() / 1000000} MB`); await inspector.heap.disable(); await fs.writeFile(filepath, JSON.stringify(profile)); logger.info(`Saved heap snapshot to ${filepath}`); } async function stopMemorySample(snapshotTriggers) { const usage = process.memoryUsage.rss() / 1000000; const profile = await inspector.heap.stopSampling(); const filepath = `traxxx_sample_${format(new Date(), 'yyyy-MM-dd_HH-mm-ss')}.heapprofile`; await inspector.heap.disable(); await fs.writeFile(filepath, JSON.stringify(profile)); logger.info(`Saved heap sample to ${filepath}`); if (usage > snapshotTriggers[0]) { await snapshotMemory(snapshotTriggers[0]); return snapshotTriggers.slice(1); } return snapshotTriggers; } async function startMemorySample(snapshotTriggers = []) { await inspector.heap.enable(); await inspector.heap.startSampling(); const usage = process.memoryUsage.rss() / 1000000; logger.info(`Start heap sampling, memory usage: ${usage} MB`); setTimeout(async () => { const newSnapshotTriggers = await stopMemorySample(snapshotTriggers); if (!done) { await startMemorySample(newSnapshotTriggers); } }, config.memorySampling.sampleDuration); } async function init() { try { if (argv.server) { await initServer(); return; } if (argv.sampleMemory) { await startMemorySample(config.memorySampling.snapshotIntervals); } if (argv.logActive) { logActive(); } if (argv.updateSearch) { await Promise.all([ updateSceneSearch(), updateMovieSearch(), ]); } if (argv.interpolateProfiles) { await interpolateProfiles(argv.interpolateProfiles.length > 0 ? argv.interpolateProfiles : null, true); } if (argv.flushActors) { await flushActors(argv.flushActors); } if (argv.flushProfiles) { await flushProfiles(argv.flushProfiles.length > 0 ? argv.flushProfiles : null); } if (argv.flushNetworks || argv.flushChannels) { await flushEntities(argv.flushNetworks, argv.flushChannels); } if (argv.flushBatches) { await flushBatches(argv.flushBatches); } if (argv.flushScenes) { await flushScenes(); } if (argv.flushMovies) { await flushMovies(); } if (argv.deleteActors) { await deleteActors(argv.deleteActors); } if (argv.deleteScenes) { await deleteScenes(argv.deleteScenes); } if (argv.deleteMovies) { await deleteMovies(argv.deleteMovies); } if (argv.flushOrphanedMedia) { await flushOrphanedMedia(); } if (argv.request) { const res = await http[argv.requestMethod](argv.request); console.log(res.status, res.body); } const actorsFromFile = argv.actorsFile && await getFileEntries(argv.actorsFile); const actorNames = (argv.actors || []).concat(actorsFromFile || []); const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames); const actorBaseScenes = argv.actors && argv.actorScenes && actors.map((actor) => actor.scenes).flat().filter(Boolean); const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates(); const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile); const sceneUrls = (argv.scene || []).concat(scenesFromFile || []); const deepScenes = argv.deep ? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])]) : [...(updateBaseScenes || []), ...(actorBaseScenes || [])]; const storedScenes = argv.save ? await storeScenes(deepScenes) : []; const moviesFromFile = argv.moviesFile && await getFileEntries(argv.moviesFile); const movieUrls = (argv.movie || []).concat(moviesFromFile || []); const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : []; const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies; const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : []; const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes; if (argv.report) { console.log(util.inspect(deepScenes, { depth: Infinity, colors: true })); console.log(util.inspect(deepMovies, { depth: Infinity, colors: true })); } if (argv.save) { const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId); const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId); await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]); } logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() })) .filter(([, value]) => value > 0) .map(([key, value]) => `${value} ${key}`) .join(', ')}`); } catch (error) { console.trace(error); logger.error(error); } await http.destroyBypassSessions(); await http.destroyBrowserSessions(); knex.destroy(); redis.disconnect(); done = true; } module.exports = init;