From 299f257a0183b676bc2cb2a485271e4645739396 Mon Sep 17 00:00:00 2001
From: DebaucheryLibrarian <moonloop.adult@protonmail.com>
Date: Mon, 27 Dec 2021 17:29:23 +0100
Subject: [PATCH] Logging memory usage on media fetch.

---
 src/app.js             |  4 ++--
 src/media.js           |  1 +
 src/scrapers/gamma.js  | 18 ++++++++++--------
 src/scrapers/kink.js   | 12 +-----------
 src/scrapers/traxxx.js |  3 ++-
 5 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/app.js b/src/app.js
index 0cfaf2f4..18c2ecf8 100644
--- a/src/app.js
+++ b/src/app.js
@@ -38,12 +38,12 @@ async function snapshotMemory() {
 	const profile = await inspector.heap.takeSnapshot();
 	const filepath = `traxxx_snapshot_${dayjs().format('YYYY-MM-DD_HH-mm-ss')}.heapprofile`;
 
-	logger.info(`Start heap snapshots, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
+	logger.info(`Starting heap snapshot, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
 
 	await inspector.heap.disable();
 	await fs.writeFile(filepath, JSON.stringify(profile));
 
-	logger.info(`Saved heap dump to ${filepath}`);
+	logger.info(`Saved heap snapshot to ${filepath}`);
 }
 
 async function stopMemorySample(snapshotTriggers) {
diff --git a/src/media.js b/src/media.js
index 8bee3c53..f1399522 100644
--- a/src/media.js
+++ b/src/media.js
@@ -581,6 +581,7 @@ async function fetchSource(source, baseMedia) {
 	const maxAttempts = source.attempts || 3;
 
 	logger.silly(`Fetching media from ${source.src}`);
+	logger.debug(`Memory usage before media fetch: ${process.memoryUsage.rss() / 1000000} MB (${source.src})`);
 
 	async function attempt(attempts = 1) {
 		const hasher = new blake2.Hash('blake2b', { digestLength: 24 });
diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js
index 78f58c22..5e69cb75 100644
--- a/src/scrapers/gamma.js
+++ b/src/scrapers/gamma.js
@@ -177,10 +177,10 @@ async function getThumbs(entryId, site, parameters) {
 	});
 
 	if (res.ok && res.body.results?.[0]?.hits[0]?.set_pictures) {
-		return res.body.results[0].hits[0].set_pictures.map((img) => ([
+		return res.body.results[0].hits[0].set_pictures.map((img) => img.thumb_path && ([
 			`https://transform.gammacdn.com/photo_set${img.thumb_path}`,
 			`https://images-evilangel.gammacdn.com/photo_set${img.thumb_path}`,
-		]));
+		])).filter(Boolean);
 	}
 
 	return [];
@@ -410,10 +410,14 @@ async function scrapeSceneApi(data, site, options) {
 
 	if (data.pictures) {
 		release.poster = [
-			`https://transform.gammacdn.com/movies${data.pictures['1920x1080']}`,
-			`https://images-evilangel.gammacdn.com/movies${data.pictures['1920x1080']}`,
-			`https://transform.gammacdn.com/movies${data.pictures.resized}`,
-			`https://images-evilangel.gammacdn.com/movies${data.pictures.resized}`,
+			...(data.pictures['1920x1080'] ? [
+				`https://transform.gammacdn.com/movies${data.pictures['1920x1080']}`,
+				`https://images-evilangel.gammacdn.com/movies${data.pictures['1920x1080']}`,
+			] : []),
+			...(data.pictures.resized ? [
+				`https://transform.gammacdn.com/movies${data.pictures.resized}`,
+				`https://images-evilangel.gammacdn.com/movies${data.pictures.resized}`,
+			] : []),
 		];
 	}
 
@@ -495,8 +499,6 @@ async function scrapeMovie({ query, el }, url, entity, baseRelease, options) {
 		release.trailer = await fetchMovieTrailer(release);
 	}
 
-	console.log(release);
-
 	return release;
 }
 
diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js
index ff26d16a..15eed3a5 100644
--- a/src/scrapers/kink.js
+++ b/src/scrapers/kink.js
@@ -140,16 +140,6 @@ async function fetchLatest(site, page = 1) {
 	return res.status;
 }
 
-async function fetchScene(url, site) {
-	const res = await qu.get(url);
-
-	if (res.ok) {
-		return scrapeScene(res.item, url, site);
-	}
-
-	return res.status;
-}
-
 async function fetchProfile({ name: actorName }, entity, include) {
 	const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
 
@@ -176,6 +166,6 @@ async function fetchProfile({ name: actorName }, entity, include) {
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
 	fetchProfile,
+	scrapeScene,
 };
diff --git a/src/scrapers/traxxx.js b/src/scrapers/traxxx.js
index c5e16bb6..ca504451 100644
--- a/src/scrapers/traxxx.js
+++ b/src/scrapers/traxxx.js
@@ -249,7 +249,8 @@ async function fetchLatest(entity, page, options) {
 				.where('is_sfw', true)
 				.pluck('path')
 				.orderByRaw('random()')
-				.limit(Math.floor(Math.random() * 10) + 1);
+				// .limit(Math.floor(Math.random() * 10) + 1)
+				.limit(100);
 
 			// const poster = 'sfw/kittens/thumbs/iNEXVlX-RLs.jpeg';