package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.204.1",
+    "version": "1.204.0",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.204.1",
+            "version": "1.204.0",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.204.1",
+    "version": "1.204.0",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/app.js

@@ -34,57 +34,47 @@ function logActive() {
 	}, typeof argv.logActive === 'number' ? argv.logActive : 60000);
 }
 
-async function snapshotMemory() {
+/*
-	const profile = await inspector.heap.takeSnapshot();
+function monitorMemory() {
-	const filepath = `traxxx_snapshot_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
+	logger.debug(`Memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
 
-	logger.info(`Start heap snapshots, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
+	if (!done) {
-
+		setTimeout(() => monitorMemory(), 10000);
-	await inspector.heap.disable();
+	}
-	await fs.writeFile(filepath, JSON.stringify(profile));
-
-	logger.info(`Saved heap dump to ${filepath}`);
 }
+*/
 
-async function stopMemorySample(snapshotTriggers) {
+async function stopMemorySample() {
 	const profile = await inspector.heap.stopSampling();
-	const filepath = `traxxx_sample_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
+	const filepath = `${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
-	const usage = process.memoryUsage.rss() / 1000000;
 
 	await inspector.heap.disable();
 	await fs.writeFile(filepath, JSON.stringify(profile));
 
 	logger.info(`Saved heap sample to ${filepath}`);
-
-	if (usage > snapshotTriggers[0]) {
-		await snapshotMemory();
-		return snapshotTriggers.slice(1);
-	}
-
-	return snapshotTriggers;
 }
 
-async function startMemorySample(snapshotTriggers = []) {
+async function startMemorySample() {
 	await inspector.heap.enable();
 	await inspector.heap.startSampling();
 
-	const usage = process.memoryUsage.rss() / 1000000;
+	// monitorMemory();
 
-	logger.info(`Start heap sampling, memory usage: ${usage} MB`);
+	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
 
 	setTimeout(async () => {
-		const newSnapshotTriggers = await stopMemorySample(snapshotTriggers);
+		await stopMemorySample();
 
 		if (!done) {
-			await startMemorySample(newSnapshotTriggers);
+			await startMemorySample();
 		}
 	}, 30000);
 }
 
 async function init() {
 	try {
-		if (argv.sampleMemory) {
+		if (argv.memory) {
-			await startMemorySample([1000]);
+			await startMemorySample();
 		}
 
 		if (argv.logActive) {
@@ -168,8 +158,6 @@ async function init() {
 			? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
 			: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
 
-		const storedScenes = argv.save ? await storeScenes(deepScenes) : [];
-
 		const sceneMovies = deepScenes ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
 		const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
 
@@ -183,9 +171,9 @@ async function init() {
 
 		if (argv.save) {
 			const storedMovies = await storeMovies(deepMovies);
-			const storedMovieScenes = await storeScenes(deepMovieScenes);
+			const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
 
-			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
+			await associateMovieScenes(storedMovies, storedScenes);
 		}
 	} catch (error) {
 		logger.error(error);

src/deep.js

@@ -54,12 +54,12 @@ function toBaseReleases(baseReleasesOrUrls, entity = null) {
 		.filter(Boolean);
 }
 
-async function fetchScene(scraper, url, entity, baseRelease, options, type = 'scene') {
+async function fetchScene(scraper, url, entity, baseRelease, options) {
-	if ((type === 'scene' && scraper.fetchScene) || (type === 'movie' && scraper.fetchMovie)) {
+	if (scraper.fetchScene) {
-		return scraper[type === 'movie' ? 'fetchMovie' : 'fetchScene'](baseRelease.url, entity, baseRelease, options, null);
+		return scraper.fetchScene(baseRelease.url, entity, baseRelease, options, null);
 	}
 
-	if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
+	if (scraper.scrapeScene) {
 		const session = qu.session();
 
 		const res = await qu.get(url, null, null, {
@@ -70,7 +70,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
 		const cookie = await session._sessionOptions.cookieJar.get(url);
 
 		if (res.ok) {
-			return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
+			return scraper.scrapeScene(res.item, url, entity, baseRelease, options, {
 				session,
 				headers: res.headers,
 				cookieJar: session._sessionOptions.cookieJar,
@@ -84,10 +84,6 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
 	return null;
 }
 
-function fetchMovie(scraper, url, entity, baseRelease, options) {
-	return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
-}
-
 async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 	const entity = baseRelease.entity || entitiesBySlug[urlToSiteSlug(baseRelease.url)];
 
@@ -110,7 +106,7 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 		return baseRelease;
 	}
 
-	if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie && !layoutScraper.scrapeMovie)) {
+	if ((type === 'scene' && !layoutScraper.fetchScene && !layoutScraper.scrapeScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
 		logger.warn(`The '${entity.name}'-scraper cannot scrape individual ${type}s`);
 		return baseRelease;
 	}
@@ -127,14 +123,12 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 		logger.debug(`Memory usage before: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
 
 		const rawScrapedRelease = type === 'scene'
-			? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options)
+			? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
-			: await fetchMovie(layoutScraper, baseRelease.url, entity, baseRelease, options);
+			: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
 
 		const pathname = new URL(baseRelease.url).pathname.replace(/\//g, '_');
 
-		if (rawScrapedRelease) {
+		delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
-			delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
-		}
 
 		if (windows.has(pathname)) {
 			logger.debug(`Closing window for ${pathname}`);

src/scrapers/gamma.js

@@ -461,7 +461,7 @@ async function fetchMovieTrailer(release) {
 	return null;
 }
 
-async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
+async function scrapeMovie({ query, el }, window, url, entity, options) {
 	const release = {};
 
 	const { dataLayer } = query.exec('//script[contains(text(), "dataLayer")]', ['dataLayer']);
@@ -495,8 +495,6 @@ async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
 		release.trailer = await fetchMovieTrailer(release);
 	}
 
-	console.log(release);
-
 	return release;
 }
 
@@ -753,6 +751,16 @@ async function fetchScene(url, site, baseRelease, options) {
 	return null;
 }
 
+async function fetchMovie(url, channel, baseRelease, options) {
+	const res = await qu.get(url, null, null);
+
+	if (res.ok) {
+		return scrapeMovie(res.item, res.window, url, channel, options);
+	}
+
+	return res.status;
+}
+
 async function fetchActorScenes(actorName, apiUrl, siteSlug) {
 	const res = await http.post(apiUrl, {
 		requests: [
@@ -847,6 +855,7 @@ module.exports = {
 	fetchApiUpcoming: fetchUpcomingApi,
 	fetchLatest,
 	fetchLatestApi,
+	fetchMovie,
 	fetchProfile,
 	fetchScene,
 	fetchSceneApi,
@@ -858,13 +867,12 @@ module.exports = {
 		fetchProfile: fetchApiProfile,
 		// fetchScene,
 		fetchScene: fetchSceneApi,
-		scrapeMovie,
+		fetchMovie,
 	},
 	getPhotos,
 	scrapeApiProfile,
 	scrapeApiReleases,
 	scrapeProfile,
 	scrapeAll,
-	scrapeMovie,
 	scrapeScene,
 };