assets/js/config/default.js

@@ -12,7 +12,6 @@ export default {
 	selectableTags: [
 		'airtight',
 		'anal',
-		'bdsm',
 		'blowbang',
 		'blowjob',
 		'creampie',

migrations/20220330230122_stats.js

@@ -1,25 +0,0 @@
-exports.up = async (knex) => knex.raw(`
-	CREATE MATERIALIZED VIEW entities_stats
-	AS
-		WITH RECURSIVE relations AS (
-			SELECT entities.id, entities.parent_id, count(releases.id) AS releases_count, count(releases.id) AS total_count
-			FROM entities
-			LEFT JOIN releases ON releases.entity_id = entities.id
-			GROUP BY entities.id
-
-			UNION ALL
-
-			SELECT entities.id AS entity_id, count(releases.id) AS releases_count, count(releases.id) + relations.total_count AS total_count
-			FROM entities
-			INNER JOIN relations ON relations.id = entities.parent_id
-			LEFT JOIN releases ON releases.entity_id = entities.id
-			GROUP BY entities.id
-		)
-
-		SELECT relations.id AS entity_id, relations.releases_count
-		FROM relations;
-`);
-
-exports.down = async (knex) => knex.raw(`
-	DROP MATERIALIZED VIEW entities_stats;
-`);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.213.4",
+    "version": "1.213.3",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.213.4",
+            "version": "1.213.3",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.213.4",
+    "version": "1.213.3",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/02_sites.js

@@ -6917,13 +6917,6 @@ const sites = [
 			tourId: 9,
 		},
 	},
-	{
-		slug: 'dpdiva',
-		name: 'DP Diva',
-		url: 'http://dpdiva.com',
-		parent: 'pervcity',
-		tags: ['dp', 'anal'],
-	},
 	// PIERRE WOODMAN
 	{
 		slug: 'woodmancastingx',

src/app.js

@@ -85,6 +85,23 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
+async function startMemorySample() {
+	await inspector.heap.enable();
+	await inspector.heap.startSampling();
+
+	// monitorMemory();
+
+	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
+
+	setTimeout(async () => {
+		await stopMemorySample();
+
+		if (!done) {
+			await startMemorySample();
+		}
+	}, 30000);
+}
+
 async function init() {
 	try {
 		if (argv.server) {

src/media.js

@@ -21,7 +21,6 @@ const argv = require('./argv');
 const knex = require('./knex');
 const http = require('./utils/http');
 const bulkInsert = require('./utils/bulk-insert');
-const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 
 const pipeline = util.promisify(stream.pipeline);
@@ -64,10 +63,10 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 		? chunks.slice(0, -1).concat(chunks.slice(-1).reverse())
 		: chunks;
 
-	const groupedMedias = lastPreferredChunks.map((mediaChunk) => {
+	const groupedMedias = lastPreferredChunks.map((chunk) => {
 		// merge chunked medias into single media with grouped fallback priorities,
 		// so the first sources of each media is preferred over all second sources, etc.
-		const sources = mediaChunk
+		const sources = chunk
 			.reduce((accSources, media) => {
 				media.sources.forEach((source, index) => {
 					if (!accSources[index]) {
@@ -83,8 +82,8 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 			.flat();
 
 		return {
-			id: mediaChunk[0].id,
+			id: chunk[0].id,
-			role: mediaChunk[0].role,
+			role: chunk[0].role,
 			sources,
 		};
 	});
@@ -236,41 +235,22 @@ async function findSourceDuplicates(baseMedias) {
 		.filter(Boolean);
 
 	const [existingSourceMedia, existingExtractMedia] = await Promise.all([
-		// my try to check thousands of URLs at once, don't pass all of them to a single query
+		knex('media').whereIn('source', sourceUrls),
-		chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
+		knex('media').whereIn('source_page', extractUrls),
-			const accUrls = await chain;
-			const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
-
-			return [...accUrls, ...existingUrls];
-		}, []),
-		chunk(extractUrls).reduce(async (chain, extractUrlsChunk) => {
-			const accUrls = await chain;
-			const existingUrls = await knex('media').whereIn('source_page', extractUrlsChunk);
-
-			return [...accUrls, ...existingUrls];
-		}, []),
 	]);
 
 	const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
 	const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
-	return {
+	return { existingSourceMediaByUrl, existingExtractMediaByUrl };
-		existingSourceMediaByUrl,
-		existingExtractMediaByUrl,
-	};
 }
 
 async function findHashDuplicates(medias) {
 	const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
 
-	const existingHashMediaEntries = await chunk(hashes, 2).reduce(async (chain, hashesChunk) => {
+	const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
-		const accHashes = await chain;
-		const existingHashes = await knex('media').whereIn('hash', hashesChunk);
-
-		return [...accHashes, ...existingHashes];
-	}, []);
-
 	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
+
 	const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
 
 	const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
@@ -620,11 +600,11 @@ async function fetchSource(source, baseMedia) {
 			const hashStream = new stream.PassThrough();
 			let size = 0;
 
-			hashStream.on('data', (streamChunk) => {
+			hashStream.on('data', (chunk) => {
-				size += streamChunk.length;
+				size += chunk.length;
 
 				if (hasherReady) {
-					hasher.write(streamChunk);
+					hasher.write(chunk);
 				}
 			});