1.217.2

.gitignore

@@ -3,6 +3,7 @@ dist/
 log/
 media/
 html/
+tmp/*
 public/js/*
 public/css/*
 config/*

assets/components/actors/tile.vue

@@ -22,7 +22,7 @@
 					class="favicon"
 				>
 					<img
-						:src="`/img/logos/${actor.entity.slug}/favicon_dark.png`"
+						:src="`/img/logos/${actor.entity.slug}/favicon_light.png`"
 						class="favicon-icon"
 					>
 				</RouterLink>

assets/components/container/container.vue

@@ -107,6 +107,7 @@ export default {
 			showSidebar: false,
 			showWarning: localStorage.getItem('consent') !== window.env.sessionId,
 			showFilters: false,
+			selected: null,
 		};
 	},
 	mounted,

assets/components/releases/release.vue

@@ -203,6 +203,19 @@
 				</div>
 			</div>
 
+			<div
+				v-if="release.qualities"
+				class="row"
+			>
+				<span class="row-label">Available qualities</span>
+
+				<span
+					v-for="quality in release.qualities"
+					:key="quality"
+					class="quality"
+				>{{ quality }}</span>
+			</div>
+
 			<div
 				v-if="release.comment"
 				class="row"
@@ -470,6 +483,16 @@ export default {
 	text-overflow: ellipsis;
 }
 
+.quality {
+	&::after {
+		content: 'p, ';
+	}
+
+	&:last-child::after {
+		content: 'p',
+	}
+}
+
 .releases {
 	margin: 0 0 .5rem 0;
 }

assets/js/config/default.js

@@ -12,6 +12,7 @@ export default {
 	selectableTags: [
 		'airtight',
 		'anal',
+		'bdsm',
 		'blowbang',
 		'blowjob',
 		'creampie',

assets/js/fragments.js

@@ -367,6 +367,7 @@ const releaseFields = `
     date
 	datePrecision
     slug
+	qualities
 	shootId
 	productionDate
 	comment
@@ -475,6 +476,7 @@ const releaseFragment = `
     duration
     createdAt
     shootId
+	qualities
 	productionDate
 	createdBatchId
 	productionLocation

config/default.js

@@ -89,6 +89,10 @@ module.exports = {
 			'uksinners',
 			// mindgeek
 			'pornhub',
+			// insex
+			'paintoy',
+			'aganmedon',
+			'sensualpain',
 		],
 		networks: [
 			// dummy network for testing

migrations/20220331135618_qualities.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.specificType('qualities', 'text[]');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.dropColumn('qualities');
+});

migrations/20220403235645_last_login.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.datetime('last_login');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.dropColumn('last_login');
+});

migrations/_20220330230122_stats.js

@@ -0,0 +1,25 @@
+exports.up = async (knex) => knex.raw(`
+	CREATE MATERIALIZED VIEW entities_stats
+	AS
+		WITH RECURSIVE relations AS (
+			SELECT entities.id, entities.parent_id, count(releases.id) AS releases_count, count(releases.id) AS total_count
+			FROM entities
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+
+			UNION ALL
+
+			SELECT entities.id AS entity_id, count(releases.id) AS releases_count, count(releases.id) + relations.total_count AS total_count
+			FROM entities
+			INNER JOIN relations ON relations.id = entities.parent_id
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+		)
+
+		SELECT relations.id AS entity_id, relations.releases_count
+		FROM relations;
+`);
+
+exports.down = async (knex) => knex.raw(`
+	DROP MATERIALIZED VIEW entities_stats;
+`);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.217.2",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.212.9",
+            "version": "1.217.2",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",
@@ -11650,25 +11650,6 @@
                 "webidl-conversions": "^3.0.0"
             }
         },
-        "node_modules/node-fetch/node_modules/tr46": {
-            "version": "0.0.3",
-            "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-            "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
-        },
-        "node_modules/node-fetch/node_modules/webidl-conversions": {
-            "version": "3.0.1",
-            "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-            "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
-        },
-        "node_modules/node-fetch/node_modules/whatwg-url": {
-            "version": "5.0.0",
-            "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-            "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
-            "dependencies": {
-                "tr46": "~0.0.3",
-                "webidl-conversions": "^3.0.0"
-            }
-        },
         "node_modules/node-gyp": {
             "version": "7.1.2",
             "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.217.2",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

public/img/logos/biphoria/misc/biphoria-light.svg

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 8192 696.7" style="enable-background:new 0 0 8192 696.7;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+</style>
+<g>
+	<path class="st0" d="M378.2,11.5c103,0,155.7,10.1,203.2,33.6c47.5,24.4,74,62.1,74,116c0,87.5-109.6,134.3-187.4,145.5v2
+		c134.6,12.2,237.6,68.2,237.6,170c0,72.2-46.2,120.1-118.8,153.7C524.8,660.8,431.1,673,334.7,673H6v-28.5
+		c103-6.1,113.6-14.2,113.6-124.1V164.2C119.6,54.2,109,46.1,13.9,40V11.5L378.2,11.5L378.2,11.5z M264.7,300.6h52.8
+		c113.6,0,184.8-40.7,184.8-126.2c0-93.6-85.8-124.1-166.3-124.1c-34.4,0-52.8,2-60.7,5.1c-10.6,4.1-10.6,21.4-10.6,45.7
+		L264.7,300.6L264.7,300.6z M264.7,337.2v181.1c0,95.6,15.8,116,100.3,114c87.1,0,176.9-37.6,176.9-142.5
+		c0-103.8-97.7-152.6-232.4-152.6L264.7,337.2L264.7,337.2z"/>
+	<path class="st0" d="M1438.2,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L1438.2,561L1438.2,561z"/>
+	<path class="st0" d="M2380.7,11.5c99,0,174.2,12.2,227,40.7c56.7,30.5,89.8,75.3,89.8,143.5c0,129.2-133.4,195.3-261.4,207.6
+		c-21.1,2-44.8,2-60.7,2l-88.4-18.4v133.3c0,109.9,10.6,118.1,132,124.1v28.5h-388.1v-28.4c100.3-6.1,110.9-14.2,110.9-124.1V164.2
+		c0-109.9-10.6-118.1-105.6-124.1V11.5L2380.7,11.5L2380.7,11.5z M2287,347.3c21.1,6.1,46.2,12.2,80.5,12.2
+		c55.4,0,169-28.5,169-162.8c0-104.8-74-146.6-184.8-146.6c-63.4,0-64.6,6.1-64.6,36.6V347.3z"/>
+	<path class="st0" d="M3690.2,412.4V331c0-75.3-7.9-79.4-93.8-83.5V219h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v229.9
+		c0,74.2,7.9,79.4,89.8,83.5v28.5h-322.1v-28.4c85.8-4.1,93.8-9.1,93.8-83.5V453.1h-269.3V561c0,74.2,7.9,79.4,85.8,83.5V673h-315.5
+		v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1c0-75.3-7.9-79.4-93.8-83.5v-28.5h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v81.4
+		L3690.2,412.4L3690.2,412.4z"/>
+	<path class="st0" d="M5085.4,438.9c0,160.8-159.7,246.2-327.4,246.2c-219.1,0-330-120.1-330-228.9c0-168.9,175.5-249.3,330-249.3
+		C4958.7,207,5085.4,310.7,5085.4,438.9z M4767.3,648.5c84.4,0,162.4-51.9,162.4-187.2c0-108.9-64.6-217.8-183.5-217.8
+		c-87.1,0-162.4,70.2-162.4,189.3C4583.8,557,4656.4,648.5,4767.3,648.5z"/>
+	<path class="st0" d="M5816.7,561c0,74.2,7.9,79.4,89.8,83.5V673h-314.1v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1
+		c0-75.3-7.9-79.4-87.1-83.5v-28.5h330c60.7,0,110.9,6.1,149.2,23.4c36.9,17.3,66,50.9,66,95.6c0,60-55.4,97.7-128,116
+		c15.8,21.4,58.1,70.2,85.8,100.7c31.7,35.6,56.7,57,72.6,68.2c18.5,12.2,43.6,23.4,66,29.5l-4,26.5h-52.8
+		c-117.5-1-153.2-26.5-188.8-63.1c-31.7-32.6-66-82.5-88.4-111c-15.8-21.4-26.4-25.5-66-25.5h-21.3V561z M5816.7,447h42.3
+		c29,0,62.1-3,85.8-14.2c36.9-17.3,52.8-47.9,52.8-83.5c0-65.1-58.1-94.6-124-94.6c-54.2,0-56.7,1-56.7,29.5L5816.7,447L5816.7,447z
+		"/>
+	<path class="st0" d="M6948,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L6948,561L6948,561z"/>
+	<path class="st0" d="M7886.5,644.5l25-2c36.9-3,43.6-10.1,21.1-54l-31.7-61.1h-180.9c-5.2,11.2-19.8,43.7-31.7,73.2
+		c-11.9,30.5-4,38.6,33,41.7l30.4,2v28.5h-238.9v-28.4c55.4-5.1,79.2-7.1,116.1-74.2l196.7-355.2l48.8-8.1l199.3,375.5
+		c30.4,56,56.7,58,112.2,62.1V673h-299.7v-28.5H7886.5z M7734.7,488.8h147.8l-72.6-144.5h-4L7734.7,488.8z"/>
+</g>
+</svg>

seeds/00_tags.js

@@ -1533,6 +1533,10 @@ const aliases = [
 		name: 'double anal (dap)',
 		for: 'dap',
 	},
+	{
+		name: 'double anal penetration',
+		for: 'dap',
+	},
 	{
 		name: 'double anal penetration (dap)',
 		for: 'dap',
@@ -1547,6 +1551,10 @@ const aliases = [
 		for: 'tvp',
 		secondary: true,
 	},
+	{
+		name: 'double vaginal penetration',
+		for: 'dvp',
+	},
 	{
 		name: 'double vaginal (dvp)',
 		for: 'dvp',

seeds/01_networks.js

@@ -366,6 +366,10 @@ const networks = [
 		name: 'Kink',
 		url: 'https://www.kink.com',
 		description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
+		parameters: {
+			interval: 1000,
+			concurrency: 1,
+		},
 	},
 	{
 		slug: 'letsdoeit',
@@ -623,7 +627,9 @@ const networks = [
 		url: 'https://www.xempire.com',
 		description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
 		parameters: {
+			layout: 'api',
 			actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
+			sceneMovies: false,
 		},
 		parent: 'gamma',
 	},

seeds/02_sites.js

@@ -1685,6 +1685,18 @@ const sites = [
 			layout: 'members',
 		},
 	},
+	// BIPHORIA
+	{
+		slug: 'biphoria',
+		name: 'BiPhoria',
+		url: 'https://www.biphoria.com',
+		independent: true,
+		tags: ['bisexual'],
+		parameters: {
+			layout: 'api',
+		},
+		parent: 'gamma',
+	},
 	// BLOWPASS
 	{
 		slug: '1000facials',
@@ -2714,164 +2726,161 @@ const sites = [
 	{
 		slug: 'blacksonblondes',
 		name: 'Blacks On Blondes',
-		url: 'https://www.blacksonblondes.com/tour',
+		url: 'https://www.blacksonblondes.com',
 		description: 'Blacks On Blondes is the Worlds Largest and Best Interracial Sex and Interracial Porn website. Black Men and White Women. BlacksOnBlondes has 23 years worth of Hardcore Interracial Content.  Featuring the entire Legendary Dogfart Movie Archive',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cuckoldsessions',
 		name: 'Cuckold Sessions',
-		url: 'https://www.cuckoldsessions.com/tour',
-		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com/tour - Hardcore Cuckold Fetish Videos',
+		url: 'https://www.cuckoldsessions.com',
+		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com - Hardcore Cuckold Fetish Videos',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryhole',
 		name: 'Glory Hole',
-		url: 'https://www.gloryhole.com/tour',
-		description: '',
+		url: 'https://www.gloryhole.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksoncougars',
 		name: 'Blacks On Cougars',
-		url: 'https://www.blacksoncougars.com/tour',
-		description: '',
+		url: 'https://www.blacksoncougars.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wefuckblackgirls',
 		name: 'We Fuck Black Girls',
 		alias: ['wfbg'],
-		url: 'https://www.wefuckblackgirls.com/tour',
-		description: '',
+		url: 'https://www.wefuckblackgirls.com',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/WeFuckBlackGirls',
+		},
 	},
 	{
 		slug: 'watchingmymomgoblack',
 		name: 'Watching My Mom Go Black',
-		url: 'https://www.watchingmymomgoblack.com/tour',
-		description: '',
+		url: 'https://www.watchingmymomgoblack.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialblowbang',
 		name: 'Interracial Blowbang',
-		url: 'https://www.interracialblowbang.com/tour',
-		description: '',
+		url: 'https://www.interracialblowbang.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cumbang',
 		name: 'Cumbang',
-		url: 'https://www.cumbang.com/tour',
-		description: '',
+		url: 'https://www.cumbang.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialpickups',
 		name: 'Interracial Pickups',
-		url: 'https://www.interracialpickups.com/tour',
-		description: '',
+		url: 'https://www.interracialpickups.com',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/InterracialPickups',
+		},
 	},
 	{
 		slug: 'watchingmydaughtergoblack',
 		name: 'Watching My Daughter Go Black',
-		url: 'https://www.watchingmydaughtergoblack.com/tour',
-		description: '',
+		url: 'https://www.watchingmydaughtergoblack.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'zebragirls',
 		name: 'Zebra Girls',
-		url: 'https://www.zebragirls.com/tour',
-		description: '',
+		url: 'https://www.zebragirls.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryholeinitiations',
 		name: 'Gloryhole Initiations',
-		url: 'https://www.gloryhole-initiations.com/tour',
-		description: '',
+		url: 'https://www.gloryhole-initiations.com',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.gloryhole-initiations.com/tourx/scenes',
+		},
 	},
 	{
 		slug: 'dogfartbehindthescenes',
 		name: 'Dogfart Behind The Scenes',
-		url: 'https://www.dogfartbehindthescenes.com/tour',
-		description: '',
+		url: 'https://www.dogfartbehindthescenes.com',
 		parent: 'dogfartnetwork',
+		tags: ['bts'],
 	},
 	{
 		slug: 'blackmeatwhitefeet',
 		name: 'Black Meat White Feet',
-		url: 'https://www.blackmeatwhitefeet.com/tour',
-		description: '',
+		url: 'https://www.blackmeatwhitefeet.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'springthomas',
 		name: 'Spring Thomas',
-		url: 'https://www.springthomas.com/tour',
-		description: '',
+		url: 'https://www.springthomas.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'katiethomas',
 		name: 'Katie Thomas',
-		url: 'https://www.katiethomas.com/tour',
-		description: '',
+		url: 'https://www.katiethomas.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'ruthblackwell',
 		name: 'Ruth Blackwell',
-		url: 'https://www.ruthblackwell.com/tour',
-		description: '',
+		url: 'https://www.ruthblackwell.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'candymonroe',
 		name: 'Candy Monroe',
-		url: 'https://www.candymonroe.com/tour',
-		description: '',
+		url: 'https://www.candymonroe.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wifewriting',
 		name: 'Wife Writing',
-		url: 'https://www.wifewriting.com/tour',
-		description: '',
+		url: 'https://www.wifewriting.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'barbcummings',
 		name: 'Barb Cummings',
-		url: 'https://www.barbcummings.com/tour',
-		description: '',
+		url: 'https://www.barbcummings.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'theminion',
 		name: 'The Minion',
-		url: 'https://www.theminion.com/tour',
-		description: '',
+		url: 'https://www.theminion.com',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksonboys',
 		name: 'Blacks On Boys',
-		url: 'https://www.blacksonboys.com/tour',
-		description: '',
+		url: 'https://www.blacksonboys.com',
 		parent: 'dogfartnetwork',
+		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys',
+		},
 	},
 	{
 		slug: 'gloryholesandhandjobs',
 		name: 'Gloryholes And Handjobs',
-		url: 'https://www.gloryholesandhandjobs.com/tour',
-		description: '',
+		url: 'https://www.gloryholesandhandjobs.com',
 		parent: 'dogfartnetwork',
+		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs',
+		},
 	},
 	// DORCEL
 	{
@@ -4219,7 +4228,6 @@ const sites = [
 		tags: ['bdsm'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.sexuallybroken.com/sb',
 		},
 	},
@@ -4230,13 +4238,20 @@ const sites = [
 		url: 'https://www.infernalrestraints.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.infernalrestraints.com/ir',
+		},
 	},
 	{
 		slug: 'hardtied',
 		name: 'Hardtied',
+		alias: ['ht'],
 		url: 'https://www.hardtied.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.hardtied.com/ht',
+		},
 	},
 	{
 		slug: 'realtimebondage',
@@ -4245,6 +4260,9 @@ const sites = [
 		url: 'https://www.realtimebondage.com',
 		tags: ['bdsm', 'live'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.realtimebondage.com/rtb',
+		},
 	},
 	{
 		slug: 'topgrl',
@@ -4254,7 +4272,6 @@ const sites = [
 		tags: ['bdsm', 'femdom'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.topgrl.com/tg',
 		},
 	},
@@ -4716,7 +4733,7 @@ const sites = [
 		slug: 'boundgangbangs',
 		name: 'Bound Gangbangs',
 		alias: ['bgb', 'bgbs'],
-		url: 'https://www.kink.com/channel/bound-gangbangs',
+		url: 'https://www.kink.com/channel/bound-gang-bangs',
 		description: 'Powerless whores tied in bondage and stuffed with a cock in every hole. At BoundGangbangs women get surprise extreme gangbangs, blindfolds, deepthroat blowjobs, sex punishment, bondage, double penetration and interracial sex.',
 		parent: 'kink',
 	},
@@ -6909,6 +6926,16 @@ const sites = [
 			tourId: 9,
 		},
 	},
+	{
+		slug: 'dpdiva',
+		name: 'DP Diva',
+		url: 'http://dpdiva.com',
+		parent: 'pervcity',
+		tags: ['dp', 'anal'],
+		parameters: {
+			native: true,
+		},
+	},
 	// PIERRE WOODMAN
 	{
 		slug: 'woodmancastingx',
@@ -8069,6 +8096,12 @@ const sites = [
 		parameters: null,
 		parent: 'realitykings',
 	},
+	// RICKYS ROOM
+	{
+		name: 'Ricky\'s Room',
+		slug: 'rickysroom',
+		url: 'https://rickysroom.com',
+	},
 	// SCORE
 	{
 		name: '18 Eighteen',
@@ -11057,6 +11090,7 @@ const sites = [
 		description: 'Watch Lesbian porn videos with the highest quality all girl on girl sex videos featuring SLAYED pornstars and models. Only the highest quality lesbian sex videos exclusive to SLAYED.com',
 		url: 'https://www.slayed.com',
 		parent: 'vixen',
+		tags: ['lesbian'],
 	},
 	// VOGOV
 	{

seeds/04_media.js

@@ -598,8 +598,10 @@ const tagMedia = [
 	['airtight', 7, 'Lana Rhoades in "Gangbang Me 3"', 'hardx'],
 	['airtight', 'hime_marie_blackedraw', 'Hime Marie', 'blackedraw'],
 	['airtight', 6, 'Remy Lacroix in "Ass Worship 14"', 'julesjordan'],
+	['airtight', 'yoha_boundgangbangs', 'Yoha in "Home Invasion"', 'boundgangbangs'],
 	['airtight', 'anissa_kate_legalporno', 'Anissa Kate in GP1962', 'analvids'],
 	['airtight', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
+	['airtight', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['airtight', 'diamond_foxxx_milfslikeitbig', 'Diamond Foxx in "Diamond\'s Bday Gangbang"', 'milfslikeitbig'],
 	['airtight', 'tory_lane_bigtitsatwork', 'Tory Lane in "I\'m Your Christmas Bonus"', 'bigtitsatwork'],
 	['airtight', 11, 'Malena Nazionale in "Rocco\'s Perverted Secretaries 2: Italian Edition"', 'roccosiffredi'],
@@ -904,6 +906,7 @@ const tagMedia = [
 	['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
 	['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
 	['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
+	['gangbang', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
 	['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
 	['gangbang', 'monika_fox_legalporno', 'Monika Fox in GL479', 'analvids'],

seeds/06_affiliates.js

@@ -100,6 +100,13 @@ const banners = [
 		channel: 'archangel',
 		tags: ['dp', 'anal', 'sex', 'interracial', 'black'],
 	},
+	{
+		id: 'wefuckblackgirls_728_90_loss',
+		width: 728,
+		height: 90,
+		network: 'dogfartnetwork',
+		tags: ['mfm', 'threesome', 'anal', 'black', 'interracial'],
+	},
 	{
 		id: 'evilangel_728_90_adriana_chechik_gangbang',
 		width: 728,
@@ -795,7 +802,7 @@ const campaigns = [
 	},
 ];
 
-exports.seed = async knex => Promise.resolve()
+exports.seed = async (knex) => Promise.resolve()
 	.then(async () => {
 		await Promise.all([
 			knex('campaigns').delete(),
@@ -810,19 +817,19 @@ exports.seed = async knex => Promise.resolve()
 		const [networks, channels, tags] = await Promise.all([
 			knex('entities')
 				.where('type', 'network')
-				.whereIn('slug', campaigns.concat(banners).map(link => link.network).filter(Boolean)),
+				.whereIn('slug', campaigns.concat(banners).map((link) => link.network).filter(Boolean)),
 			knex('entities')
 				.where('type', 'channel')
-				.whereIn('slug', campaigns.concat(banners).map(link => link.channel).filter(Boolean)),
+				.whereIn('slug', campaigns.concat(banners).map((link) => link.channel).filter(Boolean)),
 			knex('tags')
-				.whereIn('slug', banners.flatMap(banner => banner.tags || [])),
+				.whereIn('slug', banners.flatMap((banner) => banner.tags || [])),
 		]);
 
 		const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: network }), {});
 		const channelsBySlug = channels.reduce((acc, channel) => ({ ...acc, [channel.slug]: channel }), {});
 		const tagsBySlug = tags.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag }), {});
 
-		const affiliatesWithEntityId = affiliates.map(affiliate => ({
+		const affiliatesWithEntityId = affiliates.map((affiliate) => ({
 			id: affiliate.id,
 			entity_id: networksBySlug[affiliate.network]?.id || channelsBySlug[affiliate.channel]?.id || null,
 			url: affiliate.url,
@@ -830,7 +837,7 @@ exports.seed = async knex => Promise.resolve()
 			comment: affiliate.comment,
 		}));
 
-		const bannersWithEntityId = banners.map(banner => ({
+		const bannersWithEntityId = banners.map((banner) => ({
 			id: banner.id,
 			width: banner.width,
 			height: banner.height,
@@ -838,17 +845,17 @@ exports.seed = async knex => Promise.resolve()
 			entity_id: networksBySlug[banner.network]?.id || channelsBySlug[banner.channel]?.id || null,
 		}));
 
-		const bannerTags = banners.flatMap(banner => banner.tags?.map(tag => ({
+		const bannerTags = banners.flatMap((banner) => banner.tags?.map((tag) => ({
 			banner_id: banner.id,
 			tag_id: tagsBySlug[tag].id,
 		})) || []);
 
-		const campaignsWithEntityIdAndAffiliateId = campaigns.map(campaign => ({
+		const campaignsWithEntityIdAndAffiliateId = campaigns.map((campaign) => ({
 			entity_id: networksBySlug[campaign.network]?.id || channelsBySlug[campaign.channel]?.id,
 			url: campaign.url,
 			affiliate_id: campaign.affiliate,
 			banner_id: campaign.banner,
-		})).filter(link => link.entity_id && (link.url || link.affiliate_id));
+		})).filter((link) => link.entity_id && (link.url || link.affiliate_id));
 
 		await knex('affiliates').insert(affiliatesWithEntityId);
 		await bulkInsert('banners', bannersWithEntityId, false);

src/app.js

@@ -85,23 +85,6 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
-async function startMemorySample() {
-	await inspector.heap.enable();
-	await inspector.heap.startSampling();
-
-	// monitorMemory();
-
-	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
-
-	setTimeout(async () => {
-		await stopMemorySample();
-
-		if (!done) {
-			await startMemorySample();
-		}
-	}, 30000);
-}
-
 async function init() {
 	try {
 		if (argv.server) {
@@ -169,7 +152,7 @@ async function init() {
 		}
 
 		if (argv.request) {
-			const res = await http.get(argv.request);
+			const res = await http[argv.requestMethod](argv.request);
 
 			console.log(res.status, res.body);
 		}
@@ -212,6 +195,7 @@ async function init() {
 			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
 		}
 	} catch (error) {
+		console.trace(error);
 		logger.error(error);
 	}
 

src/argv.js

@@ -194,6 +194,7 @@ const { argv } = yargs
 		alias: 'pics',
 	})
 	.option('videos', {
+		alias: 'video',
 		describe: 'Include any trailers or teasers',
 		type: 'boolean',
 		default: true,
@@ -322,6 +323,16 @@ const { argv } = yargs
 		type: 'array',
 		alias: ['delete-movie', 'remove-movies', 'remove-movies'],
 	})
+	.option('request', {
+		describe: 'Make an arbitrary HTTP request',
+		type: 'string',
+	})
+	.option('request-method', {
+		alias: 'method',
+		describe: 'HTTP method for arbitrary HTTP requests',
+		type: 'string',
+		default: 'get',
+	})
 	.option('request-timeout', {
 		describe: 'Default timeout after which to cancel a HTTP request.',
 		type: 'number',

src/auth.js

@@ -34,6 +34,10 @@ async function login(credentials) {
 
 	await verifyPassword(credentials.password, user.password);
 
+	await knex('users')
+		.update('last_login', 'NOW()')
+		.where('id', user.id);
+
 	return curateUser(user);
 }
 

src/deep.js

@@ -158,8 +158,8 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 		// filter out keys with null values to ensure original base value is used instead
 		const curatedScrapedRelease = Object.entries(scrapedRelease).reduce((acc, [key, value]) => ({
 			...acc,
-			...(value !== null && value !== undefined && {
-				[key]: value,
+			...(value !== null && value !== undefined && !(Array.isArray(value) && value.filter(Boolean).length === 0) && {
+				[key]: Array.isArray(value) ? value.filter(Boolean) : value,
 			}),
 		}), {});
 

src/media.js

@@ -21,6 +21,7 @@ const argv = require('./argv');
 const knex = require('./knex');
 const http = require('./utils/http');
 const bulkInsert = require('./utils/bulk-insert');
+const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 
 const pipeline = util.promisify(stream.pipeline);
@@ -63,10 +64,10 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 		? chunks.slice(0, -1).concat(chunks.slice(-1).reverse())
 		: chunks;
 
-	const groupedMedias = lastPreferredChunks.map((chunk) => {
+	const groupedMedias = lastPreferredChunks.map((mediaChunk) => {
 		// merge chunked medias into single media with grouped fallback priorities,
 		// so the first sources of each media is preferred over all second sources, etc.
-		const sources = chunk
+		const sources = mediaChunk
 			.reduce((accSources, media) => {
 				media.sources.forEach((source, index) => {
 					if (!accSources[index]) {
@@ -82,8 +83,8 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 			.flat();
 
 		return {
-			id: chunk[0].id,
-			role: chunk[0].role,
+			id: mediaChunk[0].id,
+			role: mediaChunk[0].role,
 			sources,
 		};
 	});
@@ -235,22 +236,41 @@ async function findSourceDuplicates(baseMedias) {
 		.filter(Boolean);
 
 	const [existingSourceMedia, existingExtractMedia] = await Promise.all([
-		knex('media').whereIn('source', sourceUrls),
-		knex('media').whereIn('source_page', extractUrls),
+		// my try to check thousands of URLs at once, don't pass all of them to a single query
+		chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
+		chunk(extractUrls).reduce(async (chain, extractUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source_page', extractUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
 	]);
 
 	const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
 	const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
-	return { existingSourceMediaByUrl, existingExtractMediaByUrl };
+	return {
+		existingSourceMediaByUrl,
+		existingExtractMediaByUrl,
+	};
 }
 
 async function findHashDuplicates(medias) {
 	const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
 
-	const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
-	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
+	const existingHashMediaEntries = await chunk(hashes, 2).reduce(async (chain, hashesChunk) => {
+		const accHashes = await chain;
+		const existingHashes = await knex('media').whereIn('hash', hashesChunk);
 
+		return [...accHashes, ...existingHashes];
+	}, []);
+
+	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
 	const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
 
 	const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
@@ -600,11 +620,11 @@ async function fetchSource(source, baseMedia) {
 			const hashStream = new stream.PassThrough();
 			let size = 0;
 
-			hashStream.on('data', (chunk) => {
-				size += chunk.length;
+			hashStream.on('data', (streamChunk) => {
+				size += streamChunk.length;
 
 				if (hasherReady) {
-					hasher.write(chunk);
+					hasher.write(streamChunk);
 				}
 			});
 

src/scrapers/dogfart.js

@@ -1,153 +1,127 @@
 'use strict';
 
-/* eslint-disable newline-per-chained-call */
-// const Promise = require('bluebird');
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const qu = require('../utils/qu');
 
-async function getPhotos(albumUrl) {
-	const res = await http.get(albumUrl);
-	const html = res.body.toString();
-	const { document } = new JSDOM(html).window;
+async function getPhotos(albumUrl, channel) {
+	const res = await qu.get(albumUrl);
 
-	const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
+	if (!res.ok) {
+		return [];
+	}
+
+	const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1);
 	const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
 
 	const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
-		const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
+		const pageUrl = `${channel.url}${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
+		const networkPageUrl = `https://dogfartnetwork.com${lastPhotoPage.replace('tourx', 'tour').replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
 
-		return {
-			url: pageUrl,
-			extract: ({ query }) => query.img('.scenes-module img'),
-		};
+		const extract = ({ query }) => query.img('.scenes-module img');
+
+		return [
+			{
+				url: pageUrl,
+				extract,
+			},
+			{
+				url: networkPageUrl,
+				extract,
+			},
+		];
 	});
 
 	return photoUrls;
 }
 
-function scrapeLatest(html, site, filter = true) {
-	const { document } = new JSDOM(html).window;
-	const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
+function scrapeLatest(scenes, site, filter = true) {
+	return scenes.reduce((acc, { query }) => {
+		const release = {};
 
-	return sceneElements.map((element) => {
-		const siteUrl = element.querySelector('.recent-details-title .help-block, .model-details-title .site-name').textContent;
+		const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name');
 
-		if (filter && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
+		release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url });
+		release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`;
+
+		release.title = query.cnt('.scene-title');
+		// release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com
+
+		// release.poster = `https:${element.querySelector('img').src}`;
+		release.poster = query.img();
+		release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url');
+
+		release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
+
+		if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) {
 			// different dogfart site
-			return null;
+			return { ...acc, unextracted: [...acc.unextracted, release] };
 		}
 
-		const sceneLinkElement = element.querySelector('.thumbnail');
-		const url = qu.prefixUrl(sceneLinkElement.href, 'https://dogfartnetwork.com');
-		const { pathname } = new URL(url);
-		const entryId = `${site.slug}_${pathname.split('/')[4]}`;
-
-		const title = element.querySelector('.scene-title').textContent;
-		const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
-
-		const poster = `https:${element.querySelector('img').src}`;
-		const teaser = sceneLinkElement.dataset.preview_clip_url;
-
-		const channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
-
-		return {
-			url,
-			entryId,
-			title,
-			actors,
-			poster,
-			teaser: {
-				src: teaser,
-			},
-			site,
-			channel,
-		};
-	}).filter(Boolean);
+		return { ...acc, scenes: [...acc.scenes, release] };
+	}, {
+		scenes: [],
+		unextracted: [],
+	});
 }
 
-async function scrapeScene(html, url, site) {
-	const { document } = new JSDOM(html).window;
-
-	const title = document.querySelector('.description-title').textContent;
-	const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
-	const metaDescription = document.querySelector('meta[itemprop="description"]').content;
-	const description = metaDescription
-		? metaDescription.content
-		: document.querySelector('.description')
-			.textContent
-			.replace(/[ \t\n]{2,}/g, ' ')
-			.replace('...read more', '')
-			.trim();
-
-	const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
+async function scrapeScene({ query }, url, channel, baseScene, parameters) {
+	const release = {};
 	const { origin, pathname } = new URL(url);
-	const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
 
-	const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
-	const duration = moment
-		.duration(`00:${document
-			.querySelectorAll('.extra-info p')[1]
-			.textContent
-			.match(/\d+:\d+$/)[0]}`)
-		.asSeconds();
+	release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase();
+	release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`;
 
-	const trailerElement = document.querySelector('.html5-video');
-	const poster = `https:${trailerElement.dataset.poster}`;
-	const { trailer } = trailerElement.dataset;
+	release.title = query.cnt('.description-title') || query.text('.scene-title');
+	release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({
+		name: query.cnt(actorEl),
+		url: query.url(actorEl, null, 'href', { origin: channel.url }),
+	}));
 
-	const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0]?.href;
-	const photos = lastPhotosUrl ? await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url) : [];
+	release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description, [itemprop="description"]')?.replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim();
 
-	const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
-	const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
+	release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content');
+	release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container');
 
-	return {
-		entryId,
-		url: `${origin}${pathname}`,
-		title,
-		description,
-		actors,
-		date,
-		duration,
-		poster,
-		photos,
-		trailer: {
-			src: trailer,
-		},
-		tags,
-		rating: {
-			stars,
-		},
-		site,
-		channel,
-	};
+	release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/);
+
+	const trailer = query.video('.html5-video', 'data-trailer');
+	const lastPhotosUrl = query.urls('.pagination a').at(-1);
+
+	release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image');
+
+	if (trailer && !trailer?.includes('join')) {
+		release.trailer = trailer;
+	}
+
+	if (lastPhotosUrl && parameters.includePhotos) {
+		release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url);
+	}
+
+	release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2));
+
+	return release;
 }
 
-async function fetchLatest(site, page = 1) {
-	const res = await http.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+async function fetchLatest(channel, page = 1, { parameters }) {
+	const res = await qu.getAll(parameters.latest ? `${parameters.latest}/?p=${page}` : `${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container');
 
-	return scrapeLatest(res.body.toString(), site);
-}
+	if (res.ok) {
+		return scrapeLatest(res.items, channel);
+	}
 
-async function fetchScene(url, site) {
-	const res = await http.get(url);
-
-	return scrapeScene(res.body.toString(), url, site);
+	return res.status;
 }
 
 async function fetchProfile(baseActor, entity) {
 	const slug = slugify(baseActor.name, '+');
 	const url = `https://www.dogfartnetwork.com/tour/girls/${slug}/`;
 
-	const res = await http.get(url);
+	const res = await qu.getAll(url, '.recent-updates');
 
 	if (res.ok) {
-		const scenes = scrapeLatest(res.body, entity, false);
+		const { scenes } = scrapeLatest(res.items, entity, false);
 
+		// no bio available
 		return { scenes };
 	}
 
@@ -156,6 +130,6 @@ async function fetchProfile(baseActor, entity) {
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
 	fetchProfile,
+	scrapeScene,
 };

src/scrapers/gamma.js

@@ -457,7 +457,7 @@ async function scrapeReleaseApi(data, site, options) {
 		release.trailer = Object.entries(data.trailers).map(([quality, source]) => ({ src: source, quality }));
 	}
 
-	if (data.movie_id && !data.movie_path) {
+	if (data.movie_id && !data.movie_path && options.parameters.sceneMovies !== false) {
 		release.movie = {
 			entryId: data.movie_id,
 			title: data.movie_title,

src/scrapers/insex.js

@@ -5,6 +5,27 @@ const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
 function scrapeLatest(scenes, site) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+
+		release.title = query.cnt('.has-text-weight-bold, .is-size-6');
+		release.date = query.date('span.tag', 'YYYY-MM-DD');
+		release.actors = query.cnts('a.tag');
+
+		const cover = query.img('.image img');
+
+		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
+		release.covers = [cover];
+
+		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+
+		return release;
+	});
+}
+
+function scrapeLatestLegacy(scenes, site) {
 	return scenes.map(({ query }) => {
 		// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
 		const release = {};
@@ -47,28 +68,35 @@ function scrapeLatest(scenes, site) {
 	});
 }
 
-function scrapeLatestAlt(scenes, site) {
-	return scenes.map(({ query }) => {
-		const release = {};
+async function scrapeScene({ query }, url, channel, parameters, session) {
+	const release = {};
 
-		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+	release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
+	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
+	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
 
-		release.title = query.cnt('.has-text-weight-bold');
-		release.date = query.date('span.tag', 'YYYY-MM-DD');
-		release.actors = query.cnts('a.tag');
+	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
+	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
 
-		const cover = query.img('.image img');
+	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
+	release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
 
-		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
-		release.covers = [cover];
+	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
 
-		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
+	release.trailer = query.video();
 
-		return release;
-	});
+	if (!release.trailer && parameters.includeTrailers) {
+		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
+
+		if (trailerRes.ok) {
+			release.trailer = trailerRes.body;
+		}
+	}
+
+	return release;
 }
 
-function scrapeScene({ query }, site) {
+function scrapeSceneLegacy({ query }, site) {
 	const release = {};
 
 	const titleEl = query.q('.articleTitleText');
@@ -97,70 +125,34 @@ function scrapeScene({ query }, site) {
 	return release;
 }
 
-async function scrapeSceneAlt({ query }, url, channel, session) {
-	const release = {};
-
-	release.title = query.cnt('.columns div.is-size-5');
-	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
-	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
-
-	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
-	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
-
-	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
-	release.photos = query.imgs('body > div:nth-child(6) img');
-
-	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
-
-	release.trailer = query.video();
-
-	if (!release.trailer) {
-		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
-
-		if (trailerRes.ok) {
-			release.trailer = trailerRes.body;
-		}
-	}
-
-	return release;
-}
-
 async function fetchLatest(site, page = 1) {
-	const url = (site.parameters?.scraper === 'alt' && `${site.parameters.latest}/home.php?o=latest&p=${page}`)
-		// || (site.slug === 'paintoy' && `${site.url}/corporal/punishment/gallery.php?type=brief&page=${page}`) // paintoy's site is (was?) partially broken, use front page
-		|| `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-
-	const res = await ((site.parameters?.scraper === 'alt' && qu.getAll(url, 'body > .columns .column'))
-		// || (site.slug === 'paintoy' && qu.getAll(url, '#articleTable table[cellspacing="2"]'))
-		|| qu.get(url)); // JSON containing html as a property
+	const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
+	const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
-			return scrapeLatestAlt(res.items, site);
-		}
-
-		/*
-		if (site.slug === 'paintoy') {
-			return scrapeLatest(res.items, site);
-		}
-		*/
-
-		return scrapeLatest(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
+		return scrapeLatest(res.items, site);
 	}
 
 	return res.status;
 }
 
-async function fetchScene(url, site) {
-	const session = http.session();
-	const res = await qu.get(url, null, null, { session });
+async function fetchLatestLegacy(site, page = 1) {
+	const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
+	const res = await qu.get(url); // JSON containing html as a property
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
-			return scrapeSceneAlt(res.item, url, site, session);
-		}
+		return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
+	}
 
-		return scrapeScene(res.item, site);
+	return res.status;
+}
+
+async function fetchScene(url, site, baseRelease, parameters) {
+	const session = http.session();
+	const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, site, parameters, session);
 	}
 
 	return res.status;
@@ -169,4 +161,8 @@ async function fetchScene(url, site) {
 module.exports = {
 	fetchLatest,
 	fetchScene,
+	legacy: {
+		fetchLatest: fetchLatestLegacy,
+		scrapeScene: scrapeSceneLegacy,
+	},
 };

src/scrapers/kellymadison.js

@@ -16,7 +16,7 @@ const siteMapByKey = {
 const siteMapBySlug = Object.entries(siteMapByKey).reduce((acc, [key, value]) => ({ ...acc, [value]: key }), {});
 
 function scrapeLatest(scenes, site) {
-	return scenes.map(({ query }) => {
+	return scenes.reduce((acc, { query }) => {
 		const release = {};
 
 		release.shootId = query.q('.card-meta .text-right, .row .text-right, .card-footer-item:last-child', true);
@@ -24,11 +24,6 @@ function scrapeLatest(scenes, site) {
 		const siteId = release.shootId.match(/\d?\w{2}/)[0];
 		const siteSlug = siteMapByKey[siteId];
 
-		if (site.slug !== siteSlug) {
-			// using generic network overview, scene is not from the site we want
-			return null;
-		}
-
 		const { pathname } = new URL(query.url('h5 a, .ep-title a, .title a'));
 		[release.entryId] = pathname.match(/\d+$/);
 		release.url = `${site.url}${pathname}`;
@@ -52,8 +47,16 @@ function scrapeLatest(scenes, site) {
 			};
 		}
 
-		return release;
-	}).filter((scene) => scene);
+		if (site.slug !== siteSlug) {
+			// using generic network overview, scene is not from the site we want
+			return { ...acc, unextracted: [...acc.unextracted, release] };
+		}
+
+		return { ...acc, scenes: [...acc.scenes, release] };
+	}, {
+		scenes: [],
+		unextracted: [],
+	});
 }
 
 async function scrapeScene({ query, html }, url, baseRelease, channel, session) {

src/scrapers/kink.js

@@ -131,7 +131,18 @@ async function scrapeProfile({ query }, actorUrl, include) {
 }
 
 async function fetchLatest(site, page = 1) {
-	const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
+	// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
+	const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
+		Host: 'www.kink.com',
+		'User-Agent': 'HTTPie/2.6.0',
+		'Accept-Encoding': 'gzip, deflate, br',
+		Accept: '*/*',
+		Connection: 'keep-alive',
+
+	}, {
+		includeDefaultHeaders: false,
+		followRedirects: false,
+	});
 
 	if (res.ok) {
 		return scrapeAll(res.items, site);

src/scrapers/mindgeek.js

@@ -242,29 +242,22 @@ async function getSession(site, parameters, url) {
 	throw new Error(`Failed to acquire MindGeek session (${res.statusCode})`);
 }
 
-function scrapeProfile(data, html, releases = [], networkName) {
-	const { query } = qu.extract(html);
-
+function scrapeProfile(data, releases = [], networkName) {
 	const profile = {
 		description: data.bio,
-		aliases: data.aliases,
+		aliases: data.aliases.filter(Boolean),
 	};
 
 	profile.gender = data.gender === 'other' ? 'transsexual' : data.gender;
+	profile.measurements = data.measurements;
 
-	if (data.measurements) {
-		const [bust, waist, hip] = data.measurements.split('-');
+	profile.dateOfBirth = qu.parseDate(data.birthday);
+	profile.birthPlace = data.birthPlace;
+	profile.height = inchesToCm(data.height);
+	profile.weight = lbsToKg(data.weight);
 
-		if (profile.gender === 'female') {
-			if (bust) profile.bust = bust.toUpperCase();
-			if (waist) profile.waist = waist;
-			if (hip) profile.hip = hip;
-		}
-	}
-
-	if (data.birthPlace) profile.birthPlace = data.birthPlace;
-	if (data.height) profile.height = inchesToCm(data.height);
-	if (data.weight) profile.weight = lbsToKg(data.weight);
+	profile.hairColor = data.tags.find((tag) => /hair color/i.test(tag.category))?.name;
+	profile.ethnicity = data.tags.find((tag) => /ethnicity/i.test(tag.category))?.name;
 
 	if (data.images.card_main_rect?.[0]) {
 		profile.avatar = data.images.card_main_rect[0].xl?.url
@@ -274,9 +267,6 @@ function scrapeProfile(data, html, releases = [], networkName) {
             || data.images.card_main_rect[0].xs?.url;
 	}
 
-	const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
-	if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
-
 	if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
 		profile.naturalBoobs = true;
 	}
@@ -285,6 +275,14 @@ function scrapeProfile(data, html, releases = [], networkName) {
 		profile.naturalBoobs = false;
 	}
 
+	if (data.tags.some((tag) => /body art/i.test(tag.category) && /tattoo/i.test(tag.name))) {
+		profile.hasTattoos = true;
+	}
+
+	if (data.tags.some((tag) => /body art/i.test(tag.category) && /piercing/i.test(tag.name))) {
+		profile.hasPiercings = true;
+	}
+
 	profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName));
 
 	return profile;
@@ -377,7 +375,7 @@ async function fetchRelease(url, site, baseScene, options) {
 	return null;
 }
 
-async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, parameters }) {
+async function fetchProfile({ name: actorName }, { entity, parameters }, include) {
 	// const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
 	const { session, instanceToken } = await getSession(entity, parameters);
 
@@ -395,31 +393,22 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 		const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
 
 		if (actorData) {
-			const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;
 			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
 
-			const [actorRes, actorReleasesRes] = await Promise.all([
-				http.get(actorUrl, {
-					interval: parameters.interval,
-					concurrency: parameters.concurrency,
-				}),
-				http.get(actorReleasesUrl, {
-					session,
-					interval: parameters.interval,
-					concurrency: parameters.concurrency,
-					headers: {
-						Instance: instanceToken,
-					},
-				}),
-			]);
+			const actorReleasesRes = include.includeActorScenes && await http.get(actorReleasesUrl, {
+				session,
+				interval: parameters.interval,
+				concurrency: parameters.concurrency,
+				headers: {
+					Instance: instanceToken,
+				},
+			});
 
-			if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
-				return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, entity.slug);
+			if (actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
+				return scrapeProfile(actorData, actorReleasesRes.body.result, entity.slug);
 			}
 
-			if (actorRes.statusCode === 200) {
-				return scrapeProfile(actorData, actorRes.body.toString(), null, entity.slug);
-			}
+			return scrapeProfile(actorData, [], entity.slug);
 		}
 	}
 

src/scrapers/pervcity.js

@@ -12,6 +12,13 @@ const channelCodes = {
 	uha: 'upherasshole',
 };
 
+const qualities = {
+	v4k: 2160,
+	vFullHD: 1080,
+	vHD: 720,
+	vSD: 480,
+};
+
 const channelRegExp = new RegExp(Object.keys(channelCodes).join('|'), 'i');
 
 function scrapeAll(scenes, entity) {
@@ -36,15 +43,18 @@ function scrapeAll(scenes, entity) {
 	});
 }
 
-function scrapeScene({ query }) {
+function scrapeScene({ query }, channel) {
 	const release = {};
 
 	release.entryId = query.q('.trailerLeft img', 'id').match(/set-target-(\d+)/)[1];
 
 	release.title = query.cnt('.infoHeader h1');
-	release.description = query.cnt('.infoBox p');
+	release.description = query.cnt('.description');
+	release.duration = query.duration('.tRuntime');
 
 	release.actors = query.cnts('.infoBox .tour_update_models a');
+	release.tags = query.cnts('.tagcats a');
+	release.qualities = query.imgs('.avaiFormate img').map((src) => qualities[src.match(/\/(\w+)\.png/)[1]]).filter(Boolean);
 
 	release.poster = query.img('.posterimg');
 	release.photos = query.imgs('.trailerSnaps img').slice(1); // first photo is poster in lower quality
@@ -52,7 +62,7 @@ function scrapeScene({ query }) {
 	const trailer = query.q('script')?.textContent.match(/\/trailers\/.+\.mp4/)?.[0];
 
 	if (trailer) {
-		release.trailer = `https://pervcity.com${trailer}`;
+		release.trailer = `${channel.url}${trailer}`;
 		release.channel = channelCodes[release.trailer.match(channelRegExp)?.[0]];
 	}
 
@@ -85,15 +95,28 @@ function scrapeProfile({ query }) {
 	return profile;
 }
 
-async function fetchLatest(channel, page = 1) {
+function getLatestUrl(channel, page) {
 	if (channel.parameters?.siteId) {
-		const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
+		return `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
+	}
+
+	if (channel.parameters?.native) {
+		return `${channel.url}/search.php?site[]=&page=${page}`;
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = getLatestUrl(channel, page);
+
+	if (url) {
 		const res = await qu.getAll(url, '.videoBlock');
 
 		return res.ok ? scrapeAll(res.items, channel) : res.status;
 	}
 
-	return null;
+	return [];
 }
 
 async function fetchUpcoming(channel) {

src/scrapers/rickysroom.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const qu = require('../utils/q');
+const { lbsToKg, feetInchesToCm } = require('../utils/convert');
+
+function scrapeScene(data, channel) {
+	const release = {};
+
+	release.entryId = data.id;
+	release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
+
+	release.title = data.title;
+	release.description = data.description;
+	release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
+	release.duration = qu.durationToSeconds(data.videos_duration);
+
+	release.actors = data?.models_thumbs.map((model) => ({
+		name: model.name,
+		url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
+		avatar: model.thumb,
+	})) || data.models;
+
+	release.tags = data.tags;
+
+	release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
+	release.photos = data.previews.full
+		.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+		.concat(data.thumbs); // screenshots
+
+	release.trailer = data.trailer_url;
+	release.teaser = data.special_thumbnails;
+
+	release.qualities = data.videos && Object.values(data.videos).map((video) => video.height);
+	release.rating = data.rating;
+
+	return release;
+}
+
+function scrapeProfile(data, scenes, entity) {
+	const profile = {};
+
+	profile.entryId = data.id;
+	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
+
+	profile.description = data.Bio || data.bio;
+	profile.birthPlace = data.Born || data.born;
+	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
+
+	profile.measurements = data.Measurements || data.Measurements;
+	profile.height = feetInchesToCm(data.Height || data.height);
+	profile.weight = lbsToKg(data.Weight || data.weight);
+
+	profile.eyes = data.Eyes || data.eyes;
+	profile.hairColor = data.Hair || data.hair;
+
+	profile.avatar = data.thumb;
+
+	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
+
+	return profile;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.contents?.data) {
+			return data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url, channel, baseRelease) {
+	if (baseRelease.entryId) {
+		// deep data is identical to update data
+		return baseRelease;
+	}
+
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.content) {
+			return scrapeScene(data.props.pageProps.content, channel);
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+async function fetchProfile({ slug }, entity) {
+	const url = `${entity.url}/models/${slug}`;
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.model) {
+			return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	fetchProfile,
+};

src/scrapers/scrapers.js

@@ -53,6 +53,7 @@ const pinkyxxx = require('./pinkyxxx');
 const privateNetwork = require('./private'); // reserved keyword
 const purgatoryx = require('./purgatoryx'); // reserved keyword
 const radical = require('./radical');
+const rickysroom = require('./rickysroom');
 const score = require('./score');
 const spizoo = require('./spizoo');
 const teamskeet = require('./teamskeet');
@@ -65,7 +66,7 @@ const vixen = require('./vixen');
 const vogov = require('./vogov');
 const wankzvr = require('./wankzvr');
 const whalemember = require('./whalemember');
-const xempire = require('./xempire');
+// const xempire = require('./xempire');
 
 // profiles
 const boobpedia = require('./boobpedia');
@@ -140,6 +141,7 @@ const scrapers = {
 		private: privateNetwork,
 		purgatoryx,
 		radical,
+		rickysroom,
 		score,
 		sexyhub: mindgeek,
 		spizoo,
@@ -157,7 +159,7 @@ const scrapers = {
 		wankzvr,
 		westcoastproductions: adultempire,
 		whalemember,
-		xempire,
+		// xempire,
 	},
 	actors: {
 		'18vr': badoink,
@@ -264,6 +266,7 @@ const scrapers = {
 		purgatoryx,
 		realitykings: mindgeek,
 		realvr: badoink,
+		rickysroom,
 		roccosiffredi: famedigital,
 		score,
 		seehimfuck: hush,
@@ -291,7 +294,7 @@ const scrapers = {
 		westcoastproductions: adultempire,
 		wicked: gamma,
 		wildoncam: cherrypimps,
-		xempire,
+		xempire: gamma,
 	},
 };
 

src/scrapers/vixen.js

@@ -4,7 +4,7 @@
 const Promise = require('bluebird');
 const moment = require('moment');
 
-const logger = require('../logger')(__filename);
+const qu = require('../utils/qu');
 const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
@@ -14,34 +14,6 @@ const genderMap = {
 	T: 'transsexual', // not yet observed
 };
 
-function getPosterFallbacks(poster) {
-	return poster
-		.filter((image) => /landscape/i.test(image.name))
-		.sort((imageA, imageB) => imageB.height - imageA.height)
-		.map((image) => {
-			const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
-			// high DPI images for full HD source are huge, only prefer for smaller fallback sources
-			return image.height === 1080 ? sources : sources.reverse();
-		})
-		.flat()
-		.map((src) => ({
-			src,
-			expectType: {
-				'binary/octet-stream': 'image/jpeg',
-			},
-		}));
-}
-
-function getTeaserFallbacks(teaser) {
-	return teaser
-		.filter((video) => /landscape/i.test(video.name))
-		.map((video) => ({
-			src: video.src,
-			type: video.type,
-			quality: Number(String(video.height).replace('353', '360')),
-		}));
-}
-
 function getAvatarFallbacks(avatar) {
 	return avatar
 		.sort((imageA, imageB) => imageB.height - imageA.height)
@@ -49,6 +21,26 @@ function getAvatarFallbacks(avatar) {
 		.flat();
 }
 
+function curateSources(sources, type = 'image/jpeg') {
+	if (!sources) {
+		return null;
+	}
+
+	return sources
+		.map((source) => ({
+			src: source.src,
+			width: source.width,
+			height: source.height,
+			type: source.type || type,
+			expectType: {
+				'binary/octet-stream': type,
+			},
+		}))
+		.sort((resA, resB) => (resB.width * resB.height) - (resA.width * resA.height))			// number of pixels
+		.sort((resA, resB) => Math.abs(1.8 - Number((resA.width / resA.height).toFixed(1)))		// approximation to 16:9
+							- Math.abs(1.8 - Number((resB.width / resB.height).toFixed(1))));
+}
+
 async function getTrailer(scene, channel, url) {
 	const res = await http.post(`${channel.url}/graphql`, {
 		operationName: 'getToken',
@@ -142,158 +134,166 @@ async function getTrailer(scene, channel, url) {
 	return null;
 }
 
-/*
-async function getPhotosLegacy(url) {
-	const htmlRes = await http.get(url, {
-		extract: {
-			runScripts: 'dangerously',
-		},
-	});
-
-	try {
-		const state = htmlRes?.window?.__APOLLO_STATE__;
-
-		if (!state) {
-			return [];
-		}
-
-		const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
-		const data = state[key];
-
-		if (!data) {
-			return [];
-		}
-
-		return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
-	} catch (error) {
-		logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
-		return [];
-	}
-}
-*/
-
-async function getPhotos(url) {
-	const htmlRes = await http.get(url, {
-		parse: true,
-		extract: {
-			runScripts: 'dangerously',
-		},
-	});
-
-	try {
-		const state = htmlRes?.window?.__APOLLO_STATE__;
-
-		console.log('state', state);
-
-		if (!state) {
-			return [];
-		}
-
-		const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
-		const data = state[key];
-
-		console.log('data', data);
-
-		if (!data) {
-			return [];
-		}
-
-		console.log(data.carousel);
-
-		return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
-	} catch (error) {
-		logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
-		return [];
-	}
-}
-
-function scrapeAll(scenes, site, origin) {
-	return scenes.map((scene) => {
+function scrapeAll(scenes, channel) {
+	return scenes.map((data) => {
 		const release = {};
 
-		release.title = scene.title;
+		release.entryId = data.videoId;
+		release.url = `${channel.url}/videos/${data.slug}`;
+		release.title = data.title;
 
-		release.entryId = String(scene.newId);
-		release.url = `${site?.url || origin}/videos${scene.targetUrl}`;
+		release.date = qu.extractDate(data.releaseDate);
+		release.actors = data.modelsSlugged.map((model) => ({
+			name: model.name,
+			url: `${channel.url}/models/${model.slugged}`,
+		}));
 
-		release.date = moment.utc(scene.releaseDate).toDate();
-		release.datePrecision = 'minute';
+		release.poster = curateSources(data.images.listing);
+		release.teaser = curateSources(data.previews.listing, 'video/mp4');
 
-		release.actors = scene.models;
-		release.stars = Number(scene.textRating) / 2;
-
-		release.poster = getPosterFallbacks(scene.images.poster);
-		release.teaser = getTeaserFallbacks(scene.previews.poster);
+		release.stars = data.rating;
 
 		return release;
 	});
 }
 
 function scrapeUpcoming(scene, site) {
-	if (!scene || scene.isPreReleasePeriod) return null;
+	if (!scene || scene.isPreReleasePeriod) {
+		return null;
+	}
 
 	const release = {};
 
-	release.title = scene.targetUrl
-		.slice(1)
+	release.entryId = scene.videoId;
+	release.url = `${site.url}/videos/${scene.slug}`;
+
+	release.title = scene.slug
 		.split('-')
 		.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
 		.join(' ');
 
-	release.url = `${site.url}/videos${scene.targetUrl}`;
-
 	release.date = moment.utc(scene.releaseDate).toDate();
 	release.datePrecision = 'minute';
 
-	release.actors = scene.models;
+	release.actors = scene.models.map((model) => model.name);
 
-	release.poster = getPosterFallbacks(scene.images.poster);
-	release.teaser = getTeaserFallbacks(scene.previews.poster);
-
-	release.entryId = (release.poster[0] || release.teaser[0])?.src?.match(/\/(\d+)/)?.[1];
+	release.poster = curateSources(scene.images.poster);
+	release.teaser = curateSources(scene.previews.poster);
 
 	return [release];
 }
 
-async function scrapeScene(data, url, site, baseRelease, options) {
-	const scene = data.video;
+async function fetchGraphqlDetails(release, channel, session) {
+	const query = `
+		query($query: String!, $site: Site!) {
+		  searchVideos(input: {
+			  query: $query
+			  site: $site
+		  }) {
+			edges {
+			  node {
+				videoId
+				title
+				slug
+				description
+				releaseDate
+				categories {
+				  name
+				}
+				chapters {
+				  video {
+					title
+					seconds
+				  }
+				}
+				models {
+				  name
+				}
+				images {
+				  poster {
+					...ImageInfo
+				  }
+				}
+			  }
+			}
+		  }
+		}
 
-	const release = {
-		url,
-		title: scene.title,
-		description: scene.description,
-		actors: scene.models,
-		director: scene.directorNames,
-		duration: scene.runLength,
-		stars: scene.totalRateVal,
-		tags: scene.tags,
-	};
+		fragment ImageInfo on Image {
+		  src
+		  highdpi {
+			double
+		  }
+		}
+	`;
 
-	release.entryId = scene.newId;
+	const variables = JSON.stringify({
+		site: channel.slug.toUpperCase(),
+		query: release.title,
+	});
 
-	release.date = moment.utc(scene.releaseDate).toDate();
-	release.productionDate = moment.utc(scene.shootDate).toDate();
-	release.datePrecision = 'minute';
+	const res = await http.get(`${channel.url}/graphql?query=${encodeURI(query)}&variables=${variables}`, {
+		session,
+		headers: {
+			referer: channel.url,
+			accept: '*/*',
+		},
+	});
 
-	release.actors = baseRelease?.actors || scene.models;
-
-	release.poster = getPosterFallbacks(scene.images.poster);
-
-	// release.photos = data.pictureset.map(photo => photo.main[0]?.src).filter(Boolean);
-	if (options.includePhotos) {
-		release.photos = await getPhotos(url);
+	if (res.ok) {
+		return res.body.data?.searchVideos?.edges?.find((edge) => edge.node.videoId === release.entryId)?.node || null;
 	}
 
-	release.teaser = getTeaserFallbacks(scene.previews.poster);
+	return null;
+}
 
-	const trailer = await getTrailer(scene, site, url);
-	if (trailer) release.trailer = trailer;
+async function scrapeScene(data, url, channel, options, session) {
+	const release = {
+		url,
+		entryId: data.video.videoId || data.video.newId,
+		title: data.video.title,
+		description: data.video.description,
+		actors: data.video.models,
+		director: data.video.directorNames,
+		duration: qu.durationToSeconds(data.video.runLength),
+		stars: data.video.rating,
+	};
 
-	release.chapters = data.video.chapters?.video.map((chapter) => ({
-		tags: [chapter.title],
-		time: chapter.seconds,
+	release.entryId = data.video.newId;
+	release.date = qu.extractDate(data.video.releaseDate);
+
+	release.actors = data.video.modelsSlugged.map((model) => ({
+		name: model.name,
+		url: `${channel.url}/models/${model.slugged}`,
 	}));
 
+	release.poster = curateSources(data.video.images?.poster) || data.video.videoImage?.src;
+	release.photos = data.galleryImages?.length > 0
+		? data.galleryImages.map((image) => image.src)
+		: data.video.carousel?.map((photo) => photo.main[0]?.src).filter(Boolean);
+
+	if (options.includeTrailers) {
+		const trailer = await getTrailer(data.video, channel, url);
+
+		if (trailer) {
+			release.trailer = trailer;
+		}
+	}
+
+	release.qualities = data.video?.downloadResolutions.map((quality) => Number(quality.width)).filter(Boolean); // width property is actually the height
+
+	const graphqlDetails = await fetchGraphqlDetails(release, channel, session);
+
+	if (graphqlDetails) {
+		release.tags = graphqlDetails.categories?.map((category) => category.name);
+		release.chapters = graphqlDetails.chapters?.video?.map((chapter) => ({
+			time: chapter.seconds,
+			tags: [chapter.title],
+		}));
+	}
+
+	release.channel = data.video?.id.split(':')[0];
+
 	return release;
 }
 
@@ -347,12 +347,15 @@ async function scrapeProfile(data, origin, withReleases) {
 }
 
 async function fetchLatest(site, page = 1) {
-	const url = `${site.url}/api/videos?page=${page}`;
-	const res = await http.get(url);
+	const url = `${site.url}/videos?page=${page}`;
+	const res = await qu.get(url);
 
 	if (res.ok) {
-		if (res.body.data.videos) {
-			return scrapeAll(res.body.data.videos, site);
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data?.props.pageProps.edges) {
+			return scrapeAll(data.props.pageProps.edges.map((edge) => edge.node), site);
 		}
 
 		return [];
@@ -361,37 +364,102 @@ async function fetchLatest(site, page = 1) {
 	return res.status;
 }
 
-async function fetchUpcoming(site) {
-	const apiUrl = `${site.url}/api`;
-	const res = await http.get(apiUrl);
-
-	if (res.ok) {
-		if (res.body.data.nextScene) {
-			return scrapeUpcoming(res.body.data.nextScene, site);
+async function fetchUpcoming(channel) {
+	const query = `
+		query getNextScene($site: Site!) {
+		  nextScene: findNextReleaseVideo(input: { site: $site }) {
+			videoId
+			slug
+			isPreReleasePeriod
+			releaseDate
+			models {
+			  name
+			  __typename
+			}
+			images {
+			  countdown {
+				...ImageInfo
+				__typename
+			  }
+			  poster {
+				...ImageInfo
+				__typename
+			  }
+			  __typename
+			}
+			previews {
+			  countdown {
+				...PreviewInfo
+				__typename
+			  }
+			  poster {
+				...PreviewInfo
+				__typename
+			  }
+			  __typename
+			}
+			__typename
+		  }
 		}
 
-		return [];
-	}
+		fragment ImageInfo on Image {
+		  src
+		  placeholder
+		  width
+		  height
+		  highdpi {
+			double
+			triple
+			__typename
+		  }
+		  webp {
+			src
+			placeholder
+			highdpi {
+			  double
+			  triple
+			  __typename
+			}
+			__typename
+		  }
+		}
 
-	return res.status;
-}
+		fragment PreviewInfo on Preview {
+		  src
+		  width
+		  height
+		  type
+		}
+	`;
 
-async function fetchScene(url, site, baseRelease, options) {
-	const { origin, pathname } = new URL(url);
-	const apiUrl = `${origin}/api/${pathname.split('/').slice(-1)[0]}`;
-
-	const res = await http.get(apiUrl, {
-		extract: {
-			runScripts: 'dangerously',
+	const res = await http.post(`${channel.url}/graphql`, {
+		operationName: 'getNextScene',
+		query,
+		variables: {
+			site: channel.slug.toUpperCase(),
 		},
 	});
 
 	if (res.ok) {
-		if (res.body.data) {
-			return scrapeScene(res.body.data, url, site, baseRelease, options);
+		if (res.body.data.nextScene) {
+			return scrapeUpcoming(res.body.data.nextScene, channel);
 		}
 
-		return null;
+		return [];
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url, channel, baseRelease, options) {
+	const session = qu.session();
+	const res = await qu.get(url, null, null, { session });
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		return scrapeScene(data.props.pageProps, url, channel, options, session);
 	}
 
 	return res.status;

src/scrapers/xempire.js

@@ -1,25 +0,0 @@
-'use strict';
-
-const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');
-const qu = require('../utils/qu');
-
-async function fetchScene(url, site, baseRelease, options) {
-	const res = await qu.get(url);
-	const release = await scrapeScene(res.item, url, site, baseRelease, null, options);
-
-	const siteDomain = release.query.el('meta[name="twitter:domain"]', 'content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
-	const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
-	// const siteUrl = siteDomain && `https://www.${siteDomain}`;
-
-	release.channel = siteSlug;
-	release.director = 'Mason';
-
-	return release;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchProfile,
-	fetchUpcoming,
-	fetchScene,
-};

src/store-releases.js

@@ -39,10 +39,6 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
 		slug,
 		description: release.description,
 		comment: release.comment,
-		// director: release.director,
-		// likes: release.rating && release.rating.likes,
-		// dislikes: release.rating && release.rating.dislikes,
-		// rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
 		deep: typeof release.deep === 'boolean' ? release.deep : false,
 		deep_url: release.deepUrl,
 		updated_batch_id: batchId,
@@ -52,6 +48,7 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
 		curatedRelease.shoot_id = release.shootId || null;
 		curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null;
 		curatedRelease.duration = release.duration;
+		curatedRelease.qualities = Array.from(new Set(release.qualities?.map(Number).filter(Boolean))).sort((qualityA, qualityB) => qualityB - qualityA);
 	}
 
 	if (release.productionLocation) {

src/tools/dogfart.js

@@ -0,0 +1,76 @@
+'use strict';
+
+const fs = require('fs').promises;
+
+const Promise = require('bluebird');
+
+const qu = require('../utils/qu');
+
+const qualities = {
+	sm: 360,
+	med: 480,
+	big: 720,
+};
+
+/*
+async function scrape() {
+	const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
+		const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
+
+		if (res.ok) {
+			return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
+		}
+
+		return [];
+	}, { concurrency: 1 });
+
+	const urls = urlsByPage.flat();
+
+	await fs.writeFile('./dogfart-links', urls.join('\n'));
+
+	console.log(`Saved ${urls.length} URLs to file`);
+}
+
+async function compare() {
+	const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
+	const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
+
+	const newLinks = newLinksFile.split('\n').filter(Boolean);
+	const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
+
+	const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
+
+	await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
+
+	console.log(getLinks);
+}
+*/
+
+async function scrapeMembers() {
+	const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
+		const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
+
+		if (res.ok) {
+			return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
+				url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
+				actors: query.contents('a[href*="model.php"]'),
+				trailer: query.urls('.trailer-link a').map((url) => ({
+					src: url,
+					quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
+				})),
+			}));
+		}
+
+		return [];
+	}, { concurrency: 1 });
+
+	const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
+
+	console.log(urls);
+
+	await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
+
+	console.log(`Saved ${urls.length} URLs to file`);
+}
+
+scrapeMembers();