Merge branch 'master' into experimental

assets/js/curate.js

@@ -81,6 +81,7 @@ function curateRelease(release, type = 'scene') {
 	curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || [];
 	curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || [];
 	curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || [];
+
 	if (release.trailer) curatedRelease.trailer = release.trailer.media;
 	if (release.teaser) curatedRelease.teaser = release.teaser.media;
 	if (release.actors) curatedRelease.actors = release.actors.filter(Boolean).map((actor) => curateActor(actor.actor || actor, curatedRelease));

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.212.4",
+    "version": "1.212.9",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.212.4",
+            "version": "1.212.9",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",
@@ -11650,6 +11650,25 @@
                 "webidl-conversions": "^3.0.0"
             }
         },
+        "node_modules/node-fetch/node_modules/tr46": {
+            "version": "0.0.3",
+            "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+            "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+        },
+        "node_modules/node-fetch/node_modules/webidl-conversions": {
+            "version": "3.0.1",
+            "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+            "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+        },
+        "node_modules/node-fetch/node_modules/whatwg-url": {
+            "version": "5.0.0",
+            "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+            "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+            "dependencies": {
+                "tr46": "~0.0.3",
+                "webidl-conversions": "^3.0.0"
+            }
+        },
         "node_modules/node-gyp": {
             "version": "7.1.2",
             "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.212.4",
+    "version": "1.212.9",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

seeds/01_networks.js

@@ -412,6 +412,9 @@ const networks = [
 		url: 'https://www.milehighmedia.com',
 		description: 'MileHighMedia.com is the only niche porn network you need! Watch lesbian sex, hardcore fucking and family porn stories with the hottest teens & MILFs!',
 		parent: 'mindgeek',
+		parameters: {
+			forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API
+		},
 	},
 	{
 		slug: 'mofos',

seeds/02_sites.js

@@ -810,6 +810,13 @@ const sites = [
 		parameters: { siteId: 3261 },
 		parent: 'bang',
 	},
+	{
+		name: 'Bang! Podcast',
+		slug: 'bangpodcast',
+		url: 'https://www.bang.com/videos?in=bang!%20podcast',
+		parameters: { siteId: 6305 },
+		parent: 'bang',
+	},
 	// BANGBROS
 	{
 		name: 'Ass Parade',
@@ -5271,7 +5278,7 @@ const sites = [
 	{
 		slug: 'bigdicksatschool',
 		name: 'Big Dicks At School',
-		url: 'https://www.bigdicksatschool.com',
+		url: 'https://www.men.com/scenes?site=252',
 		description: '',
 		parameters: { siteId: 252 },
 		tags: ['gay'],
@@ -5280,7 +5287,7 @@ const sites = [
 	{
 		slug: 'drillmyhole',
 		name: 'Drill My Hole',
-		url: 'https://www.drillmyhole.com',
+		url: 'https://www.men.com/scenes?site=253',
 		description: '',
 		parameters: { siteId: 253 },
 		tags: ['gay'],
@@ -5289,7 +5296,7 @@ const sites = [
 	{
 		slug: 'str8togay',
 		name: 'Str8 to Gay',
-		url: 'https://www.str8togay.com',
+		url: 'https://www.men.com/scenes?site=254',
 		tags: ['gay'],
 		parameters: { siteId: 254 },
 		parent: 'men',
@@ -5297,7 +5304,7 @@ const sites = [
 	{
 		slug: 'thegayoffice',
 		name: 'The Gay Office',
-		url: 'https://www.thegayoffice.com',
+		url: 'https://www.men.com/scenes?site=255',
 		tags: ['gay'],
 		parameters: { siteId: 255 },
 		parent: 'men',
@@ -5305,7 +5312,7 @@ const sites = [
 	{
 		slug: 'jizzorgy',
 		name: 'Jizz Orgy',
-		url: 'https://www.jizzorgy.com',
+		url: 'https://www.men.com/scenes?site=256',
 		tags: ['gay'],
 		parameters: { siteId: 256 },
 		parent: 'men',
@@ -5313,7 +5320,7 @@ const sites = [
 	{
 		slug: 'menofuk',
 		name: 'Men of UK',
-		url: 'https://www.menofuk.com',
+		url: 'https://www.men.com/scenes?site=258',
 		tags: ['gay'],
 		parameters: { siteId: 258 },
 		parent: 'men',
@@ -5321,7 +5328,7 @@ const sites = [
 	{
 		slug: 'toptobottom',
 		name: 'Top to Bottom',
-		url: 'https://www.toptobottom.com',
+		url: 'https://www.men.com/scenes?site=259',
 		tags: ['gay'],
 		parameters: { siteId: 259 },
 		parent: 'men',
@@ -5329,7 +5336,7 @@ const sites = [
 	{
 		slug: 'godsofmen',
 		name: 'Gods of Men',
-		url: 'https://www.godsofmen.com',
+		url: 'https://www.men.com/scenes?site=260',
 		tags: ['gay'],
 		parameters: { siteId: 260 },
 		parent: 'men',
@@ -5420,7 +5427,10 @@ const sites = [
 		name: 'Doghouse Digital',
 		url: 'https://www.doghousedigital.com',
 		alias: ['dhd'],
-		parameters: { siteId: 321 },
+		parameters: {
+			siteId: 321,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5436,7 +5446,10 @@ const sites = [
 		name: 'Reality Junkies',
 		url: 'https://www.realityjunkies.com',
 		alias: ['rj'],
-		parameters: { siteId: 324 },
+		parameters: {
+			siteId: 324,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5444,7 +5457,10 @@ const sites = [
 		name: 'Sweetheart Video',
 		url: 'https://www.sweetheartvideo.com',
 		alias: ['shv'],
-		parameters: { siteId: 325 },
+		parameters: {
+			siteId: 325,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5452,7 +5468,10 @@ const sites = [
 		name: 'Sweet Sinner',
 		url: 'https://www.sweetsinner.com',
 		alias: ['ss'],
-		parameters: { siteId: 326 },
+		parameters: {
+			siteId: 326,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5461,7 +5480,10 @@ const sites = [
 		alias: ['fs'],
 		tags: ['family'],
 		url: 'https://www.familysinners.com',
-		parameters: { siteId: 317 },
+		parameters: {
+			siteId: 317,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	{
@@ -5470,7 +5492,10 @@ const sites = [
 		url: 'https://www.iconmale.com',
 		alias: ['im'],
 		tags: ['gay'],
-		parameters: { native: true, siteId: 328 },
+		parameters: {
+			siteId: 328,
+			native: true,
+		},
 		parent: 'milehighmedia',
 	},
 	// MOFOS
@@ -8586,7 +8611,10 @@ const sites = [
 		name: 'Dane Jones',
 		alias: ['dnj'],
 		url: 'https://www.danejones.com/',
-		parameters: { siteId: 290 },
+		parameters: {
+			siteId: 290,
+			native: true,
+		},
 		parent: 'sexyhub',
 	},
 	{
@@ -8594,7 +8622,10 @@ const sites = [
 		name: 'Lesbea',
 		alias: ['lsb'],
 		url: 'https://www.lesbea.com',
-		parameters: { siteId: 291 },
+		parameters: {
+			siteId: 291,
+			native: true,
+		},
 		tags: ['lesbian'],
 		parent: 'sexyhub',
 	},

src/app.js

@@ -85,6 +85,23 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
+async function startMemorySample() {
+	await inspector.heap.enable();
+	await inspector.heap.startSampling();
+
+	// monitorMemory();
+
+	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
+
+	setTimeout(async () => {
+		await stopMemorySample();
+
+		if (!done) {
+			await startMemorySample();
+		}
+	}, 30000);
+}
+
 async function init() {
 	try {
 		if (argv.server) {

src/scrapers/mindgeek.js

@@ -222,6 +222,9 @@ async function getSession(site, parameters, url) {
 
 	const res = await http.get(sessionUrl, {
 		session,
+		headers: {
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
+		},
 		interval: parameters?.interval,
 		concurrency: parameters?.concurrency,
 		parse: false,
@@ -310,6 +313,7 @@ async function fetchLatest(site, page = 1, options) {
 			Instance: instanceToken,
 			Origin: site.url,
 			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -334,6 +338,7 @@ async function fetchUpcoming(site, page, options) {
 			Instance: instanceToken,
 			Origin: site.url,
 			Referer: url,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -359,6 +364,7 @@ async function fetchRelease(url, site, baseScene, options) {
 		concurrency: options.parameters.concurrency,
 		headers: {
 			Instance: instanceToken,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});
 
@@ -381,6 +387,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 		concurrency: parameters.concurrency,
 		headers: {
 			Instance: instanceToken,
+			'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites
 		},
 	});