1.217.3

.gitignore

@@ -3,6 +3,7 @@ dist/
 log/
 media/
 html/
+tmp/*
 public/js/*
 public/css/*
 config/*

assets/components/actors/tile.vue

@@ -22,7 +22,7 @@
 					class="favicon"
 				>
 					<img
-						:src="`/img/logos/${actor.entity.slug}/favicon_dark.png`"
+						:src="`/img/logos/${actor.entity.slug}/favicon_light.png`"
 						class="favicon-icon"
 					>
 				</RouterLink>

assets/components/campaigns/campaign.vue

@@ -50,9 +50,9 @@ function ratioFilter(banner) {
 
 function entityCampaign() {
 	const bannerCampaigns = this.entity.campaigns
-		.concat(this.entity.children?.flatMap(child => child.campaigns))
+		.concat(this.entity.children?.flatMap((child) => child.campaigns))
 		.concat(this.entity.parent?.campaigns)
-		.filter(campaignX => campaignX && this.ratioFilter(campaignX.banner));
+		.filter((campaignX) => campaignX && this.ratioFilter(campaignX.banner));
 
 	if (bannerCampaigns.length > 0) {
 		const randomCampaign = bannerCampaigns[Math.floor(Math.random() * bannerCampaigns.length)];
@@ -66,7 +66,7 @@ function entityCampaign() {
 }
 
 function tagCampaign() {
-	const campaignBanners = this.tag.banners.filter(banner => banner.campaigns.length > 0 && this.ratioFilter(banner));
+	const campaignBanners = this.tag.banners.filter((banner) => banner.campaigns.length > 0 && this.ratioFilter(banner));
 	const banner = campaignBanners[Math.floor(Math.random() * campaignBanners.length)];
 
 	if (banner?.campaigns.length > 0) {
@@ -83,17 +83,25 @@ function tagCampaign() {
 	return null;
 }
 
-function campaign() {
+async function genericCampaign() {
+	const randomCampaign = await this.$store.dispatch('fetchRandomCampaign', { minRatio: this.minRatio, maxRatio: this.maxRatio });
+
+	this.campaign = randomCampaign;
+	this.$emit('campaign', randomCampaign);
+
+	return randomCampaign;
+}
+
+async function mounted() {
 	if (this.entity) {
-		return this.entityCampaign();
+		await this.entityCampaign();
 	}
 
 	if (this.tag) {
-		return this.tagCampaign();
+		await this.tagCampaign();
 	}
 
-	this.$emit('campaign', null); // allow parent to toggle campaigns depending on availability
+	await this.genericCampaign();
-	return null;
 }
 
 export default {
@@ -124,11 +132,15 @@ export default {
 		},
 	},
 	emits: ['campaign'],
-	computed: {
+	data() {
-		campaign,
+		return {
+			campaign: null,
+		};
 	},
+	mounted,
 	methods: {
 		entityCampaign,
+		genericCampaign,
 		ratioFilter,
 		tagCampaign,
 	},
@@ -139,7 +151,6 @@ export default {
 .campaign {
 	height: 100%;
 	display: inline-flex;
-	flex-grow: 1;
 	align-items: center;
 	justify-content: center;
 }

assets/components/container/container.vue

@@ -107,6 +107,7 @@ export default {
 			showSidebar: false,
 			showWarning: localStorage.getItem('consent') !== window.env.sessionId,
 			showFilters: false,
+			selected: null,
 		};
 	},
 	mounted,

assets/components/home/home.vue

@@ -1,6 +1,12 @@
 <template>
 	<div class="home">
 		<div class="content-inner">
+			<div class="campaign-container">
+				<Campaign
+					:min-ratio="6"
+				/>
+			</div>
+
 			<FilterBar
 				ref="filter"
 				:fetch-releases="fetchReleases"
@@ -32,6 +38,7 @@
 import FilterBar from '../filters/filter-bar.vue';
 import Releases from '../releases/releases.vue';
 import Pagination from '../pagination/pagination.vue';
+import Campaign from '../campaigns/campaign.vue';
 
 async function fetchReleases(scroll = true) {
 	this.done = false;
@@ -59,6 +66,7 @@ async function mounted() {
 
 export default {
 	components: {
+		Campaign,
 		FilterBar,
 		Releases,
 		Pagination,
@@ -91,4 +99,12 @@ export default {
 	flex-direction: column;
 	flex-grow: 1;
 }
+
+.campaign-container {
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	padding: .75rem 1rem .25rem 1rem;
+	background: var(--background-dim);
+}
 </style>

assets/components/releases/release.vue

@@ -203,6 +203,19 @@
 				</div>
 			</div>
 
+			<div
+				v-if="release.qualities"
+				class="row"
+			>
+				<span class="row-label">Available qualities</span>
+
+				<span
+					v-for="quality in release.qualities"
+					:key="quality"
+					class="quality"
+				>{{ quality }}</span>
+			</div>
+
 			<div
 				v-if="release.comment"
 				class="row"
@@ -470,6 +483,16 @@ export default {
 	text-overflow: ellipsis;
 }
 
+.quality {
+	&::after {
+		content: 'p, ';
+	}
+
+	&:last-child::after {
+		content: 'p',
+	}
+}
+
 .releases {
 	margin: 0 0 .5rem 0;
 }

assets/js/config/default.js

@@ -12,6 +12,7 @@ export default {
 	selectableTags: [
 		'airtight',
 		'anal',
+		'bdsm',
 		'blowbang',
 		'blowjob',
 		'creampie',

assets/js/fragments.js

@@ -367,6 +367,7 @@ const releaseFields = `
     date
 	datePrecision
     slug
+	qualities
 	shootId
 	productionDate
 	comment
@@ -475,6 +476,7 @@ const releaseFragment = `
     duration
     createdAt
     shootId
+	qualities
 	productionDate
 	createdBatchId
 	productionLocation

assets/js/ui/actions.js

@@ -227,6 +227,46 @@ function initUiActions(store, _router) {
 		};
 	}
 
+	async function fetchRandomCampaign(context, { minRatio, maxRatio }) {
+		const { randomCampaign } = await graphql(`
+			query Campaign(
+				$minRatio: BigFloat
+				$maxRatio: BigFloat
+			) {
+			  randomCampaign: getRandomCampaign(minRatio: $minRatio, maxRatio: $maxRatio) {
+				url
+				affiliate {
+				  url
+				}
+				banner {
+				  id
+				  type
+				  ratio
+				  entity {
+					type
+					slug
+					parent {
+					  type
+					  slug
+					}
+				  }
+				}
+				entity {
+				  slug
+				}
+				parent {
+				  slug
+				}
+			  }
+			}
+		`, {
+			minRatio,
+			maxRatio,
+		});
+
+		return randomCampaign;
+	}
+
 	async function fetchStats() {
 		const {
 			scenes,
@@ -273,6 +313,7 @@ function initUiActions(store, _router) {
 		setBatch,
 		setSfw,
 		setTheme,
+		fetchRandomCampaign,
 		fetchNotifications,
 		fetchStats,
 	};

config/default.js

@@ -89,6 +89,10 @@ module.exports = {
 			'uksinners',
 			// mindgeek
 			'pornhub',
+			// insex
+			'paintoy',
+			'aganmedon',
+			'sensualpain',
 		],
 		networks: [
 			// dummy network for testing

migrations/20220331135618_qualities.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.specificType('qualities', 'text[]');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('releases', (table) => {
+	table.dropColumn('qualities');
+});

migrations/20220403235645_last_login.js

@@ -0,0 +1,7 @@
+exports.up = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.datetime('last_login');
+});
+
+exports.down = async (knex) => knex.schema.alterTable('users', (table) => {
+	table.dropColumn('last_login');
+});

migrations/_20220330230122_stats.js

@@ -0,0 +1,25 @@
+exports.up = async (knex) => knex.raw(`
+	CREATE MATERIALIZED VIEW entities_stats
+	AS
+		WITH RECURSIVE relations AS (
+			SELECT entities.id, entities.parent_id, count(releases.id) AS releases_count, count(releases.id) AS total_count
+			FROM entities
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+
+			UNION ALL
+
+			SELECT entities.id AS entity_id, count(releases.id) AS releases_count, count(releases.id) + relations.total_count AS total_count
+			FROM entities
+			INNER JOIN relations ON relations.id = entities.parent_id
+			LEFT JOIN releases ON releases.entity_id = entities.id
+			GROUP BY entities.id
+		)
+
+		SELECT relations.id AS entity_id, relations.releases_count
+		FROM relations;
+`);
+
+exports.down = async (knex) => knex.raw(`
+	DROP MATERIALIZED VIEW entities_stats;
+`);

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.217.3",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.212.9",
+            "version": "1.217.3",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",
@@ -11650,25 +11650,6 @@
                 "webidl-conversions": "^3.0.0"
             }
         },
-        "node_modules/node-fetch/node_modules/tr46": {
-            "version": "0.0.3",
-            "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-            "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
-        },
-        "node_modules/node-fetch/node_modules/webidl-conversions": {
-            "version": "3.0.1",
-            "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-            "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
-        },
-        "node_modules/node-fetch/node_modules/whatwg-url": {
-            "version": "5.0.0",
-            "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-            "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
-            "dependencies": {
-                "tr46": "~0.0.3",
-                "webidl-conversions": "^3.0.0"
-            }
-        },
         "node_modules/node-gyp": {
             "version": "7.1.2",
             "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.212.9",
+    "version": "1.217.3",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

public/img/logos/biphoria/misc/biphoria-light.svg

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 8192 696.7" style="enable-background:new 0 0 8192 696.7;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+</style>
+<g>
+	<path class="st0" d="M378.2,11.5c103,0,155.7,10.1,203.2,33.6c47.5,24.4,74,62.1,74,116c0,87.5-109.6,134.3-187.4,145.5v2
+		c134.6,12.2,237.6,68.2,237.6,170c0,72.2-46.2,120.1-118.8,153.7C524.8,660.8,431.1,673,334.7,673H6v-28.5
+		c103-6.1,113.6-14.2,113.6-124.1V164.2C119.6,54.2,109,46.1,13.9,40V11.5L378.2,11.5L378.2,11.5z M264.7,300.6h52.8
+		c113.6,0,184.8-40.7,184.8-126.2c0-93.6-85.8-124.1-166.3-124.1c-34.4,0-52.8,2-60.7,5.1c-10.6,4.1-10.6,21.4-10.6,45.7
+		L264.7,300.6L264.7,300.6z M264.7,337.2v181.1c0,95.6,15.8,116,100.3,114c87.1,0,176.9-37.6,176.9-142.5
+		c0-103.8-97.7-152.6-232.4-152.6L264.7,337.2L264.7,337.2z"/>
+	<path class="st0" d="M1438.2,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L1438.2,561L1438.2,561z"/>
+	<path class="st0" d="M2380.7,11.5c99,0,174.2,12.2,227,40.7c56.7,30.5,89.8,75.3,89.8,143.5c0,129.2-133.4,195.3-261.4,207.6
+		c-21.1,2-44.8,2-60.7,2l-88.4-18.4v133.3c0,109.9,10.6,118.1,132,124.1v28.5h-388.1v-28.4c100.3-6.1,110.9-14.2,110.9-124.1V164.2
+		c0-109.9-10.6-118.1-105.6-124.1V11.5L2380.7,11.5L2380.7,11.5z M2287,347.3c21.1,6.1,46.2,12.2,80.5,12.2
+		c55.4,0,169-28.5,169-162.8c0-104.8-74-146.6-184.8-146.6c-63.4,0-64.6,6.1-64.6,36.6V347.3z"/>
+	<path class="st0" d="M3690.2,412.4V331c0-75.3-7.9-79.4-93.8-83.5V219h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v229.9
+		c0,74.2,7.9,79.4,89.8,83.5v28.5h-322.1v-28.4c85.8-4.1,93.8-9.1,93.8-83.5V453.1h-269.3V561c0,74.2,7.9,79.4,85.8,83.5V673h-315.5
+		v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1c0-75.3-7.9-79.4-93.8-83.5v-28.5h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v81.4
+		L3690.2,412.4L3690.2,412.4z"/>
+	<path class="st0" d="M5085.4,438.9c0,160.8-159.7,246.2-327.4,246.2c-219.1,0-330-120.1-330-228.9c0-168.9,175.5-249.3,330-249.3
+		C4958.7,207,5085.4,310.7,5085.4,438.9z M4767.3,648.5c84.4,0,162.4-51.9,162.4-187.2c0-108.9-64.6-217.8-183.5-217.8
+		c-87.1,0-162.4,70.2-162.4,189.3C4583.8,557,4656.4,648.5,4767.3,648.5z"/>
+	<path class="st0" d="M5816.7,561c0,74.2,7.9,79.4,89.8,83.5V673h-314.1v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1
+		c0-75.3-7.9-79.4-87.1-83.5v-28.5h330c60.7,0,110.9,6.1,149.2,23.4c36.9,17.3,66,50.9,66,95.6c0,60-55.4,97.7-128,116
+		c15.8,21.4,58.1,70.2,85.8,100.7c31.7,35.6,56.7,57,72.6,68.2c18.5,12.2,43.6,23.4,66,29.5l-4,26.5h-52.8
+		c-117.5-1-153.2-26.5-188.8-63.1c-31.7-32.6-66-82.5-88.4-111c-15.8-21.4-26.4-25.5-66-25.5h-21.3V561z M5816.7,447h42.3
+		c29,0,62.1-3,85.8-14.2c36.9-17.3,52.8-47.9,52.8-83.5c0-65.1-58.1-94.6-124-94.6c-54.2,0-56.7,1-56.7,29.5L5816.7,447L5816.7,447z
+		"/>
+	<path class="st0" d="M6948,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L6948,561L6948,561z"/>
+	<path class="st0" d="M7886.5,644.5l25-2c36.9-3,43.6-10.1,21.1-54l-31.7-61.1h-180.9c-5.2,11.2-19.8,43.7-31.7,73.2
+		c-11.9,30.5-4,38.6,33,41.7l30.4,2v28.5h-238.9v-28.4c55.4-5.1,79.2-7.1,116.1-74.2l196.7-355.2l48.8-8.1l199.3,375.5
+		c30.4,56,56.7,58,112.2,62.1V673h-299.7v-28.5H7886.5z M7734.7,488.8h147.8l-72.6-144.5h-4L7734.7,488.8z"/>
+</g>
+</svg>

seeds/00_tags.js

@@ -1533,6 +1533,10 @@ const aliases = [
 		name: 'double anal (dap)',
 		for: 'dap',
 	},
+	{
+		name: 'double anal penetration',
+		for: 'dap',
+	},
 	{
 		name: 'double anal penetration (dap)',
 		for: 'dap',
@@ -1547,6 +1551,10 @@ const aliases = [
 		for: 'tvp',
 		secondary: true,
 	},
+	{
+		name: 'double vaginal penetration',
+		for: 'dvp',
+	},
 	{
 		name: 'double vaginal (dvp)',
 		for: 'dvp',

seeds/01_networks.js

@@ -366,6 +366,10 @@ const networks = [
 		name: 'Kink',
 		url: 'https://www.kink.com',
 		description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
+		parameters: {
+			interval: 1000,
+			concurrency: 1,
+		},
 	},
 	{
 		slug: 'letsdoeit',
@@ -623,7 +627,9 @@ const networks = [
 		url: 'https://www.xempire.com',
 		description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
 		parameters: {
+			layout: 'api',
 			actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
+			sceneMovies: false,
 		},
 		parent: 'gamma',
 	},

seeds/02_sites.js

@@ -1685,6 +1685,18 @@ const sites = [
 			layout: 'members',
 		},
 	},
+	// BIPHORIA
+	{
+		slug: 'biphoria',
+		name: 'BiPhoria',
+		url: 'https://www.biphoria.com',
+		independent: true,
+		tags: ['bisexual'],
+		parameters: {
+			layout: 'api',
+		},
+		parent: 'gamma',
+	},
 	// BLOWPASS
 	{
 		slug: '1000facials',
@@ -2714,164 +2726,161 @@ const sites = [
 	{
 		slug: 'blacksonblondes',
 		name: 'Blacks On Blondes',
-		url: 'https://www.blacksonblondes.com/tour',
+		url: 'https://www.blacksonblondes.com',
 		description: 'Blacks On Blondes is the Worlds Largest and Best Interracial Sex and Interracial Porn website. Black Men and White Women. BlacksOnBlondes has 23 years worth of Hardcore Interracial Content.  Featuring the entire Legendary Dogfart Movie Archive',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cuckoldsessions',
 		name: 'Cuckold Sessions',
-		url: 'https://www.cuckoldsessions.com/tour',
+		url: 'https://www.cuckoldsessions.com',
-		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com/tour - Hardcore Cuckold Fetish Videos',
+		description: 'Dogfart, the #1 Interracial Network in the World Presents CuckoldSessions.com - Hardcore Cuckold Fetish Videos',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryhole',
 		name: 'Glory Hole',
-		url: 'https://www.gloryhole.com/tour',
+		url: 'https://www.gloryhole.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksoncougars',
 		name: 'Blacks On Cougars',
-		url: 'https://www.blacksoncougars.com/tour',
+		url: 'https://www.blacksoncougars.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wefuckblackgirls',
 		name: 'We Fuck Black Girls',
 		alias: ['wfbg'],
-		url: 'https://www.wefuckblackgirls.com/tour',
+		url: 'https://www.wefuckblackgirls.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/WeFuckBlackGirls',
+		},
 	},
 	{
 		slug: 'watchingmymomgoblack',
 		name: 'Watching My Mom Go Black',
-		url: 'https://www.watchingmymomgoblack.com/tour',
+		url: 'https://www.watchingmymomgoblack.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialblowbang',
 		name: 'Interracial Blowbang',
-		url: 'https://www.interracialblowbang.com/tour',
+		url: 'https://www.interracialblowbang.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'cumbang',
 		name: 'Cumbang',
-		url: 'https://www.cumbang.com/tour',
+		url: 'https://www.cumbang.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'interracialpickups',
 		name: 'Interracial Pickups',
-		url: 'https://www.interracialpickups.com/tour',
+		url: 'https://www.interracialpickups.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/InterracialPickups',
+		},
 	},
 	{
 		slug: 'watchingmydaughtergoblack',
 		name: 'Watching My Daughter Go Black',
-		url: 'https://www.watchingmydaughtergoblack.com/tour',
+		url: 'https://www.watchingmydaughtergoblack.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'zebragirls',
 		name: 'Zebra Girls',
-		url: 'https://www.zebragirls.com/tour',
+		url: 'https://www.zebragirls.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'gloryholeinitiations',
 		name: 'Gloryhole Initiations',
-		url: 'https://www.gloryhole-initiations.com/tour',
+		url: 'https://www.gloryhole-initiations.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		parameters: {
+			latest: 'https://www.gloryhole-initiations.com/tourx/scenes',
+		},
 	},
 	{
 		slug: 'dogfartbehindthescenes',
 		name: 'Dogfart Behind The Scenes',
-		url: 'https://www.dogfartbehindthescenes.com/tour',
+		url: 'https://www.dogfartbehindthescenes.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		tags: ['bts'],
 	},
 	{
 		slug: 'blackmeatwhitefeet',
 		name: 'Black Meat White Feet',
-		url: 'https://www.blackmeatwhitefeet.com/tour',
+		url: 'https://www.blackmeatwhitefeet.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'springthomas',
 		name: 'Spring Thomas',
-		url: 'https://www.springthomas.com/tour',
+		url: 'https://www.springthomas.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'katiethomas',
 		name: 'Katie Thomas',
-		url: 'https://www.katiethomas.com/tour',
+		url: 'https://www.katiethomas.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'ruthblackwell',
 		name: 'Ruth Blackwell',
-		url: 'https://www.ruthblackwell.com/tour',
+		url: 'https://www.ruthblackwell.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'candymonroe',
 		name: 'Candy Monroe',
-		url: 'https://www.candymonroe.com/tour',
+		url: 'https://www.candymonroe.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'wifewriting',
 		name: 'Wife Writing',
-		url: 'https://www.wifewriting.com/tour',
+		url: 'https://www.wifewriting.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'barbcummings',
 		name: 'Barb Cummings',
-		url: 'https://www.barbcummings.com/tour',
+		url: 'https://www.barbcummings.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'theminion',
 		name: 'The Minion',
-		url: 'https://www.theminion.com/tour',
+		url: 'https://www.theminion.com',
-		description: '',
 		parent: 'dogfartnetwork',
 	},
 	{
 		slug: 'blacksonboys',
 		name: 'Blacks On Boys',
-		url: 'https://www.blacksonboys.com/tour',
+		url: 'https://www.blacksonboys.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/BlacksOnBoys',
+		},
 	},
 	{
 		slug: 'gloryholesandhandjobs',
 		name: 'Gloryholes And Handjobs',
-		url: 'https://www.gloryholesandhandjobs.com/tour',
+		url: 'https://www.gloryholesandhandjobs.com',
-		description: '',
 		parent: 'dogfartnetwork',
+		tags: ['gay'],
+		parameters: {
+			latest: 'https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs',
+		},
 	},
 	// DORCEL
 	{
@@ -4219,7 +4228,6 @@ const sites = [
 		tags: ['bdsm'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.sexuallybroken.com/sb',
 		},
 	},
@@ -4230,13 +4238,20 @@ const sites = [
 		url: 'https://www.infernalrestraints.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.infernalrestraints.com/ir',
+		},
 	},
 	{
 		slug: 'hardtied',
 		name: 'Hardtied',
+		alias: ['ht'],
 		url: 'https://www.hardtied.com',
 		tags: ['bdsm'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.hardtied.com/ht',
+		},
 	},
 	{
 		slug: 'realtimebondage',
@@ -4245,6 +4260,9 @@ const sites = [
 		url: 'https://www.realtimebondage.com',
 		tags: ['bdsm', 'live'],
 		parent: 'insex',
+		parameters: {
+			latest: 'https://www.realtimebondage.com/rtb',
+		},
 	},
 	{
 		slug: 'topgrl',
@@ -4254,7 +4272,6 @@ const sites = [
 		tags: ['bdsm', 'femdom'],
 		parent: 'insex',
 		parameters: {
-			scraper: 'alt',
 			latest: 'https://www.topgrl.com/tg',
 		},
 	},
@@ -4716,7 +4733,7 @@ const sites = [
 		slug: 'boundgangbangs',
 		name: 'Bound Gangbangs',
 		alias: ['bgb', 'bgbs'],
-		url: 'https://www.kink.com/channel/bound-gangbangs',
+		url: 'https://www.kink.com/channel/bound-gang-bangs',
 		description: 'Powerless whores tied in bondage and stuffed with a cock in every hole. At BoundGangbangs women get surprise extreme gangbangs, blindfolds, deepthroat blowjobs, sex punishment, bondage, double penetration and interracial sex.',
 		parent: 'kink',
 	},
@@ -6909,6 +6926,16 @@ const sites = [
 			tourId: 9,
 		},
 	},
+	{
+		slug: 'dpdiva',
+		name: 'DP Diva',
+		url: 'http://dpdiva.com',
+		parent: 'pervcity',
+		tags: ['dp', 'anal'],
+		parameters: {
+			native: true,
+		},
+	},
 	// PIERRE WOODMAN
 	{
 		slug: 'woodmancastingx',
@@ -8069,6 +8096,12 @@ const sites = [
 		parameters: null,
 		parent: 'realitykings',
 	},
+	// RICKYS ROOM
+	{
+		name: 'Ricky\'s Room',
+		slug: 'rickysroom',
+		url: 'https://rickysroom.com',
+	},
 	// SCORE
 	{
 		name: '18 Eighteen',
@@ -11057,6 +11090,7 @@ const sites = [
 		description: 'Watch Lesbian porn videos with the highest quality all girl on girl sex videos featuring SLAYED pornstars and models. Only the highest quality lesbian sex videos exclusive to SLAYED.com',
 		url: 'https://www.slayed.com',
 		parent: 'vixen',
+		tags: ['lesbian'],
 	},
 	// VOGOV
 	{

seeds/04_media.js

@@ -598,8 +598,10 @@ const tagMedia = [
 	['airtight', 7, 'Lana Rhoades in "Gangbang Me 3"', 'hardx'],
 	['airtight', 'hime_marie_blackedraw', 'Hime Marie', 'blackedraw'],
 	['airtight', 6, 'Remy Lacroix in "Ass Worship 14"', 'julesjordan'],
+	['airtight', 'yoha_boundgangbangs', 'Yoha in "Home Invasion"', 'boundgangbangs'],
 	['airtight', 'anissa_kate_legalporno', 'Anissa Kate in GP1962', 'analvids'],
 	['airtight', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
+	['airtight', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['airtight', 'diamond_foxxx_milfslikeitbig', 'Diamond Foxx in "Diamond\'s Bday Gangbang"', 'milfslikeitbig'],
 	['airtight', 'tory_lane_bigtitsatwork', 'Tory Lane in "I\'m Your Christmas Bonus"', 'bigtitsatwork'],
 	['airtight', 11, 'Malena Nazionale in "Rocco\'s Perverted Secretaries 2: Italian Edition"', 'roccosiffredi'],
@@ -904,6 +906,7 @@ const tagMedia = [
 	['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
 	['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
 	['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
+	['gangbang', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
 	['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
 	['gangbang', 'monika_fox_legalporno', 'Monika Fox in GL479', 'analvids'],

seeds/06_affiliates.js

@@ -1,3 +1,5 @@
+const fs = require('fs').promises;
+const path = require('path');
 const bulkInsert = require('../src/utils/bulk-insert');
 
 const affiliates = [
@@ -15,445 +17,190 @@ const affiliates = [
 	},
 ];
 
-const banners = [
+const bannerTags = {
-	{
+	// 21sextury
-		id: '21sextury_300_250_anal',
+	'21sextury_300_250_anal': ['anal', 'blonde'],
-		width: 300,
+	'21sextury_770_76_gina_gerson_dp': ['dp', 'anal', 'mfm', 'blonde'],
-		height: 250,
+	'21sextury_770_76_veronica_leal_dp': ['dp', 'anal', 'mfm', 'blowjob', 'blonde'],
-		network: '21sextury',
+	'21sextreme_300_250_cum': ['facial', 'cum-in-mouth'],
-		tags: ['anal', 'blonde'],
+	'21naturals_315_300': ['brunette', 'natural-boobs'],
-	},
+	'21naturals_315_300_1': ['blonde', 'natural-boobs'],
-	{
+	'21naturals_315_300_gina_gerson': ['sex', 'blonde', 'natural-boobs'],
-		id: '21sextury_770_76_gina_gerson_dp',
+	'21naturals_315_300_ginebra_bellucci': ['sex', 'brunette', 'natural-boobs'],
-		width: 770,
+	'21naturals_315_300_lana_roy_anal': ['anal', 'brunette', 'natural-boobs'],
-		height: 76,
+	'21naturals_770_76_alexis_crystal': ['blowjob', 'blonde'],
-		network: '21sextury',
+	'21naturals_970_90': ['sex', 'brunette'],
-		tags: ['dp', 'anal', 'mfm', 'blonde'],
+	// archangel
-	},
+	archangel_970_90_kendra_lust: ['dp', 'anal', 'sex', 'interracial', 'black'],
-	{
+	// dogfart
-		id: '21sextury_770_76_veronica_leal_dp',
+	wefuckblackgirls_728_90_loss: ['mfm', 'threesome', 'anal', 'black', 'interracial'],
-		width: 770,
+	// evilangel
-		height: 76,
+	evilangel_728_90_adriana_chechik_gangbang: ['gangbang', 'airtight', 'dp', 'dvp', 'facial', 'brunette'],
-		network: '21sextury',
+	evilangel_728_90_kenzie_reeves_lexi_lore: ['anal', 'mff', 'blowjob', 'blonde'],
-		tags: ['dp', 'anal', 'mfm', 'blowjob', 'blonde'],
+	evilangel_970_90_one_dollar: ['sex', 'mff'],
-	},
+	// julesjordan
-	{
+	julesjordan_728_90_jill_kassidy: ['sex', 'blowjob', 'black-cock', 'brunette'],
-		id: '21sextreme_300_250_cum',
+	julesjordan_728_90_angela_white: ['sex', 'black-cock', 'brunette'],
-		width: 300,
+	julesjordan_728_90_adriana_chechik: ['anal', 'black-cock', 'brunette'],
-		height: 250,
+	julesjordan_728_90_autumn_falls: ['sex', 'big-boobs', 'brunette'],
-		network: '21sextreme',
+	julesjordan_728_90_gabbie_carter: ['sex', 'blowjob', 'facefucking', 'big-boobs', 'brunette'],
-		tags: ['facial', 'cum-in-mouth'],
+	manuelferrara_728_90_asses: ['big-butt'],
-	},
+	// kink
-	{
+	boundgangbangs_305_99_moretta_11975_animated: ['gangbang', 'mfm', 'bdsm', 'blonde'],
-		id: '21naturals_315_300',
+	boundgangbangs_305_99_moretta_11975: ['blowbang', 'blowjob', 'bdsm', 'blonde'],
-		width: 315,
+	boundgangbangs_315_300_lou_charmelle_12402_animated: ['gangbang', 'airtight', 'dp', 'bdsm', 'bondage'],
-		height: 300,
+	boundgangbangs_315_300_lou_charmelle_12402: ['gangbang', 'mfm', 'bdsm', 'bondage'],
-		network: '21naturals',
+	boundgangbangs_770_76_amy_brooke_11965_animated: ['gangbang', 'airtight', 'mfm', 'bdsm', 'bondage'],
-		tags: ['brunette', 'natural-boobs'],
+	boundgangbangs_770_76_anissa_kate_19662: ['gangbang', 'airtight', 'bdsm', 'bondage'],
-	},
+	boundgangbangs_970_90_sasha_swift_18815: ['gangbang', 'blowbang', 'dp', 'blowjob', 'facefucking', 'facial', 'bdsm'],
-	{
+	boundgangbangs_970_90_skylar_price_12403_animated: ['gangbang', 'mfm', 'blowbang', 'blowjob', 'blonde', 'bdsm'],
-		id: '21naturals_315_300_1',
+	hardcoregangbang_300_250_kira_noir_44157: ['blowbang', 'black', 'bdsm'],
-		width: 315,
+	hardcoregangbang_305_99_kira_noir: ['blowbang', 'black', 'bdsm'],
-		height: 300,
+	hardcoregangbang_900_250_gloves_blonde: ['blowbang', 'blonde', 'bdsm'],
-		network: '21naturals',
+	hardcoregangbang_1000_100: ['gangbang', 'mfm', 'bdsm'],
-		tags: ['blonde', 'natural-boobs'],
+	// teenmegaworld
-	},
+	analangels_468_80_animated: ['anal'],
-	{
+	analangels_300_250_animated: ['anal'],
-		id: '21naturals_315_300_gina_gerson',
+	analbeauty_468_80_animated: ['anal'],
-		width: 315,
+	analbeauty_300_250_animated: ['anal'],
-		height: 300,
+	analbeauty_300_250_tail_animated: ['anal', 'bondage', 'bdsm'],
-		network: '21naturals',
+	beautyangels_468_80_animated: ['solo'],
-		tags: ['sex', 'blonde', 'natural-boobs'],
+	beautyangels_300_250_69_animated: ['lesbian', '69'],
-	},
+	beautyangels_300_250_lesbian_animated: ['lesbian'],
-	{
+	teenmegaworld_300_250_animated: ['solo'],
-		id: '21naturals_315_300_ginebra_bellucci',
+	tmwvrnet_468_80_animated: ['vr'],
-		width: 315,
+	// legalporno/analvids/pornworld
-		height: 300,
+	pornworld_600_120_1: ['anal', 'brunette'],
-		network: '21naturals',
+	pornworld_600_120_2: ['mfm', 'sex', 'brunette'],
-		tags: ['sex', 'brunette', 'natural-boobs'],
+	// xempire
-	},
+	hardx_770_76_anal: ['anal', 'blonde'],
-	{
+	hardx_770_76_esperanza_anal: ['anal', 'brunette'],
-		id: '21naturals_315_300_lana_roy_anal',
+	hardx_770_76_zoey_monroe_mff: ['sex', 'mff', 'blonde'],
-		width: 315,
+	xempire_315_300: ['blowbang', 'sex', 'black-cock', 'brunette'],
-		height: 300,
+	xempire_970_90_mff: ['mff', '69', 'brunette'],
-		network: '21naturals',
+	// vixen
-		tags: ['anal', 'brunette', 'natural-boobs'],
+	blacked_300_250_cherry_kiss_dp: ['dp', 'anal', 'black-cock'],
-	},
+	blacked_300_250_cherry_kiss_anal_mfm: ['anal', 'black-cock'],
-	{
+	tushy_970_70_alexa_flexy_dp: ['dp', 'anal'],
-		id: '21naturals_770_76_alexis_crystal',
+	tushy_776_70_gianna_dior_anal: ['anal'],
-		width: 770,
+};
-		height: 76,
+
-		network: '21naturals',
+/*
-		tags: ['blowjob', 'blonde'],
+const bannerActors = {
-	},
+	// 21sextury
-	{
+	'21sextury_770_76_gina_gerson_dp': ['gina-gerson'],
-		id: '21naturals_970_90',
+	'21sextury_770_76_veronica_leal_dp': ['veronica-leal'],
-		width: 970,
+	'21naturals_315_300_gina_gerson': ['gina-gerson'],
-		height: 90,
+	'21naturals_315_300_ginebra_bellucci': ['ginebra-bellucci'],
-		network: '21naturals',
+	'21naturals_315_300_lana_roy_anal': ['lana-roy'],
-		tags: ['sex', 'brunette'],
+	'21naturals_770_76_alexis_crystal': ['alexis-crystal'],
-	},
+	// archangel
-	{
+	archangel_970_90_kendra_lust: ['kendra-lust'],
-		id: 'archangel_970_90_kendra_lust',
+	// evilangel
-		width: 970,
+	evilangel_728_90_adriana_chechik_gangbang: ['adriana-chechik'],
-		height: 90,
+	evilangel_728_90_kenzie_reeves_lexi_lore: ['kenzie-reeves', 'lexi-lore'],
-		channel: 'archangel',
+	// julesjordan
-		tags: ['dp', 'anal', 'sex', 'interracial', 'black'],
+	julesjordan_728_90_jill_kassidy: ['jill-kassidy'],
-	},
+	julesjordan_728_90_angela_white: ['angela-white'],
-	{
+	julesjordan_728_90_adriana_chechik: ['adriana-chechik'],
-		id: 'evilangel_728_90_adriana_chechik_gangbang',
+	julesjordan_728_90_autumn_falls: ['autumn-falls'],
-		width: 728,
+	julesjordan_728_90_gabbie_carter: ['gabbie-carter'],
-		height: 90,
+	// kink
-		network: 'evilangel',
+	boundgangbangs_305_99_moretta_11975_animated: ['moretta'],
-		tags: ['gangbang', 'airtight', 'dp', 'dvp', 'facial', 'brunette'],
+	boundgangbangs_305_99_moretta_11975: ['moretta'],
-	},
+	boundgangbangs_315_300_lou_charmelle_12402_animated: ['lou-charmelle'],
-	{
+	boundgangbangs_315_300_lou_charmelle_12402: ['lou-charmelle'],
-		id: 'evilangel_728_90_kenzie_reeves_lexi_lore',
+	boundgangbangs_770_76_amy_brooke_11965_animated: ['amy-brooke'],
-		width: 728,
+	boundgangbangs_770_76_anissa_kate_19662: ['anissa-kate'],
-		height: 90,
+	boundgangbangs_970_90_sasha_swift_18815: ['sasha-swift'],
-		network: 'evilangel',
+	boundgangbangs_970_90_skylar_price_12403_animated: ['skylar-price'],
-		tags: ['anal', 'mff', 'blowjob', 'blonde'],
+	hardcoregangbang_300_250_kira_noir_44157: ['kira-noir'],
-	},
+	hardcoregangbang_305_99_kira_noir: ['kira-noir'],
-	{
+	// xempire
-		id: 'evilangel_970_90_one_dollar',
+	hardx_770_76_esperanza_anal: ['esperanza-del-horno'],
-		width: 970,
+	hardx_770_76_zoey_monroe_mff: ['zoey-monroe'],
-		height: 90,
+	// vixen
-		network: 'evilangel',
+	blacked_300_250_cherry_kiss_dp: ['cherry-kiss'],
-		tags: ['sex', 'mff'],
+	blacked_300_250_cherry_kiss_anal_mfm: ['cherry-kiss'],
-	},
+	tushy_970_70_alexa_flexy_dp: ['alexa-flexy'],
-	{
+	tushy_776_70_gianna_dior_anal: ['gianna-dior'],
-		id: 'hardx_770_76_anal',
+};
-		width: 770,
+*/
-		height: 76,
-		channel: 'hardx',
-		tags: ['anal', 'blonde'],
-	},
-	{
-		id: 'hardx_770_76_esperanza_anal',
-		width: 770,
-		height: 76,
-		channel: 'hardx',
-		tags: ['anal', 'brunette'],
-	},
-	{
-		id: 'hardx_770_76_zoey_monroe_mff',
-		width: 770,
-		height: 76,
-		channel: 'hardx',
-		tags: ['sex', 'mff', 'blonde'],
-	},
-	{
-		id: 'julesjordan_728_90_jill_kassidy',
-		width: 728,
-		height: 90,
-		network: 'julesjordan',
-		tags: ['sex', 'blowjob', 'black-cock', 'brunette'],
-	},
-	{
-		id: 'julesjordan_728_90_angela_white',
-		width: 728,
-		height: 90,
-		network: 'julesjordan',
-		tags: ['sex', 'black-cock', 'brunette'],
-	},
-	{
-		id: 'julesjordan_728_90_adriana_chechik',
-		width: 728,
-		height: 90,
-		network: 'julesjordan',
-		tags: ['anal', 'black-cock', 'brunette'],
-	},
-	{
-		id: 'julesjordan_728_90_autumn_falls',
-		width: 728,
-		height: 90,
-		network: 'julesjordan',
-		tags: ['sex', 'big-boobs', 'brunette'],
-	},
-	{
-		id: 'julesjordan_728_90_gabbie_carter',
-		width: 728,
-		height: 90,
-		network: 'julesjordan',
-		tags: ['sex', 'blowjob', 'facefucking', 'big-boobs', 'brunette'],
-	},
-	{
-		id: 'boundgangbangs_305_99_moretta_11975_animated',
-		width: 305,
-		height: 99,
-		type: 'gif',
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'mfm', 'bdsm', 'blonde'],
-	},
-	{
-		id: 'boundgangbangs_305_99_moretta_11975',
-		width: 305,
-		height: 99,
-		channel: 'boundgangbangs',
-		tags: ['blowbang', 'blowjob', 'bdsm', 'blonde'],
-	},
-	{
-		id: 'boundgangbangs_315_300_lou_charmelle_12402_animated',
-		width: 315,
-		height: 300,
-		type: 'gif',
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'airtight', 'dp', 'bdsm', 'bondage'],
-	},
-	{
-		id: 'boundgangbangs_315_300_lou_charmelle_12402',
-		width: 315,
-		height: 300,
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'mfm', 'bdsm', 'bondage'],
-	},
-	{
-		id: 'boundgangbangs_770_76_amy_brooke_11965',
-		width: 770,
-		height: 76,
-		type: 'gif',
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'airtight', 'mfm', 'bdsm', 'bondage'],
-	},
-	{
-		id: 'boundgangbangs_770_76_anissa_kate_19662',
-		width: 770,
-		height: 76,
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'airtight', 'bdsm', 'bondage'],
-	},
-	{
-		id: 'boundgangbangs_970_90_sasha_swift_18815',
-		width: 970,
-		height: 90,
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'blowbang', 'dp', 'blowjob', 'facefucking', 'facial', 'bdsm'],
-	},
-	{
-		id: 'boundgangbangs_970_90_skylar_price_12403',
-		width: 970,
-		height: 90,
-		type: 'gif',
-		channel: 'boundgangbangs',
-		tags: ['gangbang', 'mfm', 'blowbang', 'blowjob', 'blonde', 'bdsm'],
-	},
-	{
-		id: 'hardcoregangbang_300_250_kira_noir_44157',
-		width: 300,
-		height: 250,
-		channel: 'hardcoregangbang',
-		tags: ['blowbang', 'black', 'bdsm'],
-	},
-	{
-		id: 'hardcoregangbang_305_99_kira_noir',
-		width: 305,
-		height: 99,
-		channel: 'hardcoregangbang',
-		tags: ['blowbang', 'black', 'bdsm'],
-	},
-	{
-		id: 'hardcoregangbang_900_250_gloves_blonde',
-		width: 900,
-		height: 250,
-		channel: 'hardcoregangbang',
-		tags: ['blowbang', 'blonde', 'bdsm'],
-	},
-	{
-		id: 'hardcoregangbang_1000_100',
-		width: 1000,
-		height: 100,
-		channel: 'hardcoregangbang',
-		tags: ['gangbang', 'mfm', 'bdsm'],
-	},
-	{
-		id: 'manuelferrara_728_90_asses',
-		width: 728,
-		height: 90,
-		channel: 'manuelferrara',
-		tags: ['big-butt'],
-	},
-	{
-		id: 'analangels_468_80_animated',
-		width: 468,
-		height: 80,
-		type: 'gif',
-		channel: 'analangels',
-		tags: ['anal'],
-	},
-	{
-		id: 'analangels_300_250_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		channel: 'analangels',
-		tags: ['anal'],
-	},
-	{
-		id: 'analbeauty_468_80_animated',
-		width: 468,
-		height: 80,
-		type: 'gif',
-		channel: 'analbeauty',
-		tags: ['anal'],
-	},
-	{
-		id: 'analbeauty_300_250_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		channel: 'analbeauty',
-		tags: ['anal'],
-	},
-	{
-		id: 'analbeauty_300_250_tail_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		channel: 'analbeauty',
-		tags: ['anal', 'bondage', 'bdsm'],
-	},
-	{
-		id: 'beautyangels_468_80_animated',
-		width: 468,
-		height: 80,
-		type: 'gif',
-		channel: 'beautyangels',
-		tags: ['solo'],
-	},
-	{
-		id: 'beautyangels_300_250_69_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		channel: 'beautyangels',
-		tags: ['lesbian', '69'],
-	},
-	{
-		id: 'beautyangels_300_250_lesbian_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		channel: 'beautyangels',
-		tags: ['lesbian'],
-	},
-	{
-		id: 'teenmegaworld_300_250_animated',
-		width: 300,
-		height: 250,
-		type: 'gif',
-		network: 'teenmegaworld',
-		tags: ['solo'],
-	},
-	{
-		id: 'tmwvrnet_468_80_animated',
-		width: 468,
-		height: 80,
-		type: 'gif',
-		channel: 'tmwvrnet',
-		tags: ['vr'],
-	},
-	{
-		id: 'pornworld_600_120_1',
-		width: 600,
-		height: 120,
-		network: 'pornworld',
-		tags: ['anal', 'brunette'],
-	},
-	{
-		id: 'pornworld_600_120_2',
-		width: 600,
-		height: 120,
-		network: 'pornworld',
-		tags: ['mfm', 'sex', 'brunette'],
-	},
-	{
-		id: 'xempire_315_300',
-		width: 315,
-		height: 300,
-		network: 'xempire',
-		tags: ['blowbang', 'sex', 'black-cock', 'brunette'],
-	},
-	{
-		id: 'xempire_970_90_mff',
-		width: 970,
-		height: 90,
-		network: 'xempire',
-		tags: ['mff', '69', 'brunette'],
-	},
-];
 
 const campaigns = [
+	// 21sextury
 	{
 		network: '21sextury',
 		url: 'https://www.iyalc.com/21sextury/go.php?pr=8&su=1&si=207&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21sextury',
 		banner: '21sextury_300_250_anal',
+		network: '21sextury',
 		url: 'https://www.iyalc.com/21sextury/go.php?pr=8&su=1&si=207&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21sextury',
 		banner: '21sextury_770_76_gina_gerson_dp',
-		url: 'https://www.iyalc.com/21sextury/go.php?pr=8&su=1&si=207&ad=277470&pa=index&ar=&buffer=',
-		comment: 'per signup',
-	},
-	{
 		network: '21sextury',
-		banner: '21sextury_770_76_veronica_leal_dp',
 		url: 'https://www.iyalc.com/21sextury/go.php?pr=8&su=1&si=207&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21sextreme',
+		banner: '21sextury_770_76_veronica_leal_dp',
+		network: '21sextury',
+		url: 'https://www.iyalc.com/21sextury/go.php?pr=8&su=1&si=207&ad=277470&pa=index&ar=&buffer=',
+		comment: 'per signup',
+	},
+	{
 		banner: '21sextreme_300_250_cum',
+		network: '21sextreme',
 		url: 'https://www.iyalc.com/21sextreme/go.php?pr=8&su=1&si=208&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_315_300',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_315_300_1',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_315_300_gina_gerson',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_315_300_ginebra_bellucci',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_315_300_lana_roy_anal',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_770_76_alexis_crystal',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: '21naturals',
 		banner: '21naturals_970_90',
+		network: '21naturals',
 		url: 'https://www.iyalc.com/21naturals/go.php?pr=8&su=1&si=209&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
+	// archangel
 	{
 		channel: 'archangel',
 		affiliate: 'archangel_share',
@@ -474,94 +221,94 @@ const campaigns = [
 		comment: 'per signup',
 	},
 	{
-		network: 'evilangel',
 		banner: 'evilangel_728_90_adriana_chechik_gangbang',
+		network: 'evilangel',
 		url: 'https://www.iyalc.com/evilangel/go.php?pr=8&su=2&si=128&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		network: 'evilangel',
 		banner: 'evilangel_728_90_kenzie_reeves_lexi_lore',
-		url: 'https://www.iyalc.com/evilangel/go.php?pr=8&su=2&si=128&ad=277470&pa=index&ar=&buffer=',
-		comment: 'per signup',
-	},
-	{
 		network: 'evilangel',
-		banner: 'evilangel_970_90_one_dollar',
 		url: 'https://www.iyalc.com/evilangel/go.php?pr=8&su=2&si=128&ad=277470&pa=index&ar=&buffer=',
 		comment: 'per signup',
 	},
 	{
-		channel: 'hardx',
+		banner: 'evilangel_970_90_one_dollar',
-		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
+		network: 'evilangel',
-		comment: '$30 per signup',
+		url: 'https://www.iyalc.com/evilangel/go.php?pr=8&su=2&si=128&ad=277470&pa=index&ar=&buffer=',
-	},
+		comment: 'per signup',
-	{
-		channel: 'hardx',
-		banner: 'hardx_770_76_anal',
-		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
-		comment: '$30 per signup',
-	},
-	{
-		channel: 'hardx',
-		banner: 'hardx_770_76_esperanza_anal',
-		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
-		comment: '$30 per signup',
-	},
-	{
-		channel: 'hardx',
-		banner: 'hardx_770_76_zoey_monroe_mff',
-		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
-		comment: '$30 per signup',
 	},
+	// julesjordan
 	{
 		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
 	{
-		network: 'julesjordan',
 		banner: 'julesjordan_728_90_jill_kassidy',
+		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
 	{
-		network: 'julesjordan',
 		banner: 'julesjordan_728_90_angela_white',
+		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
 	{
-		network: 'julesjordan',
 		banner: 'julesjordan_728_90_adriana_chechik',
+		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
 	{
-		network: 'julesjordan',
 		banner: 'julesjordan_728_90_autumn_falls',
+		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
 	{
-		network: 'julesjordan',
 		banner: 'julesjordan_728_90_gabbie_carter',
+		network: 'julesjordan',
 		url: 'https://enter.julesjordan.com/track/Mzk3MS4yLjMuNi4wLjAuMC4wLjA',
 		comment: '$30 per signup',
 	},
+	{
+		channel: 'manuelferrara',
+		url: 'https://enter.manuelferrara.com/track/Mzk3MS4yLjcuMTYuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	{
+		banner: 'manuelferrara_728_90_asses',
+		channel: 'manuelferrara',
+		url: 'https://enter.manuelferrara.com/track/Mzk3MS4yLjcuMTYuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'theassfactory',
+		url: 'https://enter.theassfactory.com/track/Mzk3MS4yLjEuMS4wLjAuMC4wLjA',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'spermswallowers',
+		url: 'https://enter.spermswallowers.com/track/Mzk3MS4yLjUuMTMuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	// kink
 	{
 		network: 'kink',
 		affiliate: 'kink_params',
 		comment: '50%',
 	},
 	{
-		url: 'https://www.kink.com/channel/bound-gang-bangs?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		channel: 'boundgangbangs',
+		url: 'https://www.kink.com/channel/bound-gang-bangs?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		comment: '50%',
 	},
 	{
-		url: 'https://www.kink.com/channel/hardcore-gangbang?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		channel: 'hardcoregangbang',
+		url: 'https://www.kink.com/channel/hardcore-gangbang?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		comment: '50%',
 	},
 	{
@@ -589,7 +336,7 @@ const campaigns = [
 		comment: '50%',
 	},
 	{
-		banner: 'boundgangbangs_770_76_amy_brooke_11965',
+		banner: 'boundgangbangs_770_76_amy_brooke_11965_animated',
 		url: 'https://www.kink.com/channel/bound-gang-bangs?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		channel: 'boundgangbangs',
 		comment: '50%',
@@ -607,7 +354,7 @@ const campaigns = [
 		comment: '50%',
 	},
 	{
-		banner: 'boundgangbangs_970_90_skylar_price_12403',
+		banner: 'boundgangbangs_970_90_skylar_price_12403_animated',
 		url: 'https://www.kink.com/channel/bound-gang-bangs?t=eyJhZmZpbGlhdGUiOiJEZWJhdWNoZXJ5TGliIiwiY2FtcGFpZ24iOiJkZWZhdWx0IiwiYWdldmVyaWZpZWQiOiJ5In0',
 		channel: 'boundgangbangs',
 		comment: '50%',
@@ -636,6 +383,7 @@ const campaigns = [
 		channel: 'hardcoregangbang',
 		comment: '50%',
 	},
+	// kellymadison/teenfidelity
 	{
 		network: 'kellymadison',
 		url: 'https://www2.kellymadison.com/track/MTAxOTE0LjYuMS4xLjAuMC4wLjAuMA',
@@ -656,22 +404,7 @@ const campaigns = [
 		url: 'https://www2.teenfidelity.com/track/MTAxOTE0LjYuNS42LjAuMC4wLjAuMA',
 		comment: '$25 per signup',
 	},
-	{
+	// teenmegaworld
-		channel: 'manuelferrara',
-		url: 'https://enter.manuelferrara.com/track/Mzk3MS4yLjcuMTYuMC4wLjAuMC4w',
-		comment: '$30 per signup',
-	},
-	{
-		channel: 'manuelferrara',
-		banner: 'manuelferrara_728_90_asses',
-		url: 'https://enter.manuelferrara.com/track/Mzk3MS4yLjcuMTYuMC4wLjAuMC4w',
-		comment: '$30 per signup',
-	},
-	{
-		channel: 'spermswallowers',
-		url: 'https://enter.spermswallowers.com/track/Mzk3MS4yLjUuMTMuMC4wLjAuMC4w',
-		comment: '$30 per signup',
-	},
 	{
 		network: 'teenmegaworld',
 		url: 'https://secure.teenmegaworld.net/track/MzAxNjcxLjUuMS4xLjAuMC4wLjAuMA',
@@ -737,11 +470,7 @@ const campaigns = [
 		url: 'https://secure.tmwvrnet.com/track/MzAxNjcxLjUuNDQuNDQuMC4wLjAuMC4w',
 		comment: 'recurring',
 	},
-	{
+	// legalporno/analvids/pornworld
-		channel: 'theassfactory',
-		url: 'https://enter.theassfactory.com/track/Mzk3MS4yLjEuMS4wLjAuMC4wLjA',
-		comment: '$30 per signup',
-	},
 	{
 		network: 'analvids',
 		url: 'https://www.analvids.com/new-videos?aff=BW90MHT1DP____',
@@ -776,6 +505,7 @@ const campaigns = [
 		url: 'https://pornworld.com/new-videos?aff=BW90MHT1DP____',
 		comment: 'default offer',
 	},
+	// xempire
 	{
 		network: 'xempire',
 		url: 'https://www.blazinglink.com/xempire/go.php?pr=12&su=2&si=81&pa=index&ar=&ad=277470',
@@ -793,10 +523,109 @@ const campaigns = [
 		url: 'https://www.blazinglink.com/xempire/go.php?pr=12&su=2&si=81&pa=index&ar=&ad=277470',
 		comment: '$30 per signup',
 	},
+	{
+		channel: 'hardx',
+		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'hardx',
+		banner: 'hardx_770_76_anal',
+		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'hardx',
+		banner: 'hardx_770_76_esperanza_anal',
+		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'hardx',
+		banner: 'hardx_770_76_zoey_monroe_mff',
+		url: 'https://www.blazinglink.com/hardx/go.php?pr=12&su=2&si=68&pa=index&ar=&ad=277470',
+		comment: '$30 per signup',
+	},
+	// vixen
+	{
+		channel: 'blacked',
+		url: 'https://join.blacked.com/track/MTA0MS43OC4zLjMuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'blacked',
+		banner: 'blacked_300_250_cherry_kiss_dp',
+		url: 'https://join.blacked.com/track/MTA0MS43OC4zLjMuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'blacked',
+		banner: 'blacked_300_250_cherry_kiss_anal_mfm',
+		url: 'https://join.blacked.com/track/MTA0MS43OC4zLjMuMC4wLjAuMC4w',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'tushy',
+		url: 'https://join.tushy.com/track/MTA0MS43OC43LjIwLjAuMC4wLjAuMA',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'tushy',
+		banner: 'tushy_970_70_alexa_flexy_dp',
+		url: 'https://join.tushy.com/track/MTA0MS43OC43LjIwLjAuMC4wLjAuMA',
+		comment: '$30 per signup',
+	},
+	{
+		channel: 'tushy',
+		banner: 'tushy_776_70_gianna_dior_anal',
+		url: 'https://join.tushy.com/track/MTA0MS43OC43LjIwLjAuMC4wLjAuMA',
+		comment: '$30 per signup',
+	},
 ];
 
-exports.seed = async knex => Promise.resolve()
+exports.seed = async (knex) => Promise.resolve()
 	.then(async () => {
+		const bannerNetworks = await fs.readdir('./public/img/banners');
+
+		// derive entity, width and height from filepath to minimize redundant work
+		const rawBanners = await Promise.all(bannerNetworks.map(async (network) => {
+			const networkPaths = await fs.readdir(`./public/img/banners/${network}`);
+
+			return Promise.all(networkPaths.map(async (file) => {
+				if (file.charAt(0) === '_') {
+					return null;
+				}
+
+				if (await fs.stat(`./public/img/banners/${network}/${file}`).then(async (stats) => stats.isDirectory())) {
+					const channelPaths = await fs.readdir(`./public/img/banners/${network}/${file}`);
+
+					return channelPaths.map((filepath) => ({
+						id: path.parse(filepath).name,
+						channel: file,
+					}));
+				}
+
+				return {
+					id: path.parse(file).name,
+					network,
+				};
+			}));
+		}));
+
+		const banners = rawBanners
+			.flat(2)
+			.filter(Boolean)
+			.map((banner) => {
+				const [, width, height] = banner.id.match(/[a-z0-9]+_(\d+)_(\d+)/);
+
+				return {
+					...banner,
+					width: Number(width),
+					height: Number(height),
+					tags: bannerTags[banner.id] || [],
+				};
+			});
+
 		await Promise.all([
 			knex('campaigns').delete(),
 			knex('banners_tags').delete(),
@@ -810,19 +639,23 @@ exports.seed = async knex => Promise.resolve()
 		const [networks, channels, tags] = await Promise.all([
 			knex('entities')
 				.where('type', 'network')
-				.whereIn('slug', campaigns.concat(banners).map(link => link.network).filter(Boolean)),
+				.whereIn('slug', campaigns.concat(banners).map((link) => link.network).filter(Boolean)),
 			knex('entities')
 				.where('type', 'channel')
-				.whereIn('slug', campaigns.concat(banners).map(link => link.channel).filter(Boolean)),
+				.whereIn('slug', campaigns.concat(banners).map((link) => link.channel).filter(Boolean)),
 			knex('tags')
-				.whereIn('slug', banners.flatMap(banner => banner.tags || [])),
+				.whereIn('slug', banners.flatMap((banner) => banner.tags || [])),
+			/*
+			knex('actors')
+				.whereIn('slug', banners.flatMap((banner) => banner.tags || [])),
+			*/
 		]);
 
 		const networksBySlug = networks.reduce((acc, network) => ({ ...acc, [network.slug]: network }), {});
 		const channelsBySlug = channels.reduce((acc, channel) => ({ ...acc, [channel.slug]: channel }), {});
 		const tagsBySlug = tags.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag }), {});
 
-		const affiliatesWithEntityId = affiliates.map(affiliate => ({
+		const affiliatesWithEntityId = affiliates.map((affiliate) => ({
 			id: affiliate.id,
 			entity_id: networksBySlug[affiliate.network]?.id || channelsBySlug[affiliate.channel]?.id || null,
 			url: affiliate.url,
@@ -830,28 +663,28 @@ exports.seed = async knex => Promise.resolve()
 			comment: affiliate.comment,
 		}));
 
-		const bannersWithEntityId = banners.map(banner => ({
+		const bannersWithEntityId = banners.map((banner) => ({
 			id: banner.id,
 			width: banner.width,
 			height: banner.height,
-			type: banner.type,
+			type: banner.type === 'gif' || banner.id.includes('animated') ? 'gif' : 'jpg',
-			entity_id: networksBySlug[banner.network]?.id || channelsBySlug[banner.channel]?.id || null,
+			entity_id: networksBySlug[banner.network]?.id || channelsBySlug[banner.channel]?.id || channelsBySlug[banner.network]?.id || null,
 		}));
 
-		const bannerTags = banners.flatMap(banner => banner.tags?.map(tag => ({
+		const bannerTagEntries = banners.flatMap((banner) => banner.tags?.map((tag) => ({
 			banner_id: banner.id,
 			tag_id: tagsBySlug[tag].id,
 		})) || []);
 
-		const campaignsWithEntityIdAndAffiliateId = campaigns.map(campaign => ({
+		const campaignsWithEntityIdAndAffiliateId = campaigns.map((campaign) => ({
 			entity_id: networksBySlug[campaign.network]?.id || channelsBySlug[campaign.channel]?.id,
 			url: campaign.url,
 			affiliate_id: campaign.affiliate,
 			banner_id: campaign.banner,
-		})).filter(link => link.entity_id && (link.url || link.affiliate_id));
+		})).filter((link) => link.entity_id && (link.url || link.affiliate_id));
 
 		await knex('affiliates').insert(affiliatesWithEntityId);
 		await bulkInsert('banners', bannersWithEntityId, false);
-		await bulkInsert('banners_tags', bannerTags, false);
+		await bulkInsert('banners_tags', bannerTagEntries, false);
 		await bulkInsert('campaigns', campaignsWithEntityIdAndAffiliateId, false);
 	});

src/app.js

@@ -85,23 +85,6 @@ async function startMemorySample(snapshotTriggers = []) {
 	}, config.memorySampling.sampleDuration);
 }
 
-async function startMemorySample() {
-	await inspector.heap.enable();
-	await inspector.heap.startSampling();
-
-	// monitorMemory();
-
-	logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`);
-
-	setTimeout(async () => {
-		await stopMemorySample();
-
-		if (!done) {
-			await startMemorySample();
-		}
-	}, 30000);
-}
-
 async function init() {
 	try {
 		if (argv.server) {
@@ -169,7 +152,7 @@ async function init() {
 		}
 
 		if (argv.request) {
-			const res = await http.get(argv.request);
+			const res = await http[argv.requestMethod](argv.request);
 
 			console.log(res.status, res.body);
 		}
@@ -212,6 +195,7 @@ async function init() {
 			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
 		}
 	} catch (error) {
+		console.trace(error);
 		logger.error(error);
 	}
 

src/argv.js

@@ -194,6 +194,7 @@ const { argv } = yargs
 		alias: 'pics',
 	})
 	.option('videos', {
+		alias: 'video',
 		describe: 'Include any trailers or teasers',
 		type: 'boolean',
 		default: true,
@@ -322,6 +323,16 @@ const { argv } = yargs
 		type: 'array',
 		alias: ['delete-movie', 'remove-movies', 'remove-movies'],
 	})
+	.option('request', {
+		describe: 'Make an arbitrary HTTP request',
+		type: 'string',
+	})
+	.option('request-method', {
+		alias: 'method',
+		describe: 'HTTP method for arbitrary HTTP requests',
+		type: 'string',
+		default: 'get',
+	})
 	.option('request-timeout', {
 		describe: 'Default timeout after which to cancel a HTTP request.',
 		type: 'number',

src/auth.js

@@ -34,6 +34,10 @@ async function login(credentials) {
 
 	await verifyPassword(credentials.password, user.password);
 
+	await knex('users')
+		.update('last_login', 'NOW()')
+		.where('id', user.id);
+
 	return curateUser(user);
 }
 

src/deep.js

@@ -158,8 +158,8 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
 		// filter out keys with null values to ensure original base value is used instead
 		const curatedScrapedRelease = Object.entries(scrapedRelease).reduce((acc, [key, value]) => ({
 			...acc,
-			...(value !== null && value !== undefined && {
+			...(value !== null && value !== undefined && !(Array.isArray(value) && value.filter(Boolean).length === 0) && {
-				[key]: value,
+				[key]: Array.isArray(value) ? value.filter(Boolean) : value,
 			}),
 		}), {});
 

src/media.js

@@ -21,6 +21,7 @@ const argv = require('./argv');
 const knex = require('./knex');
 const http = require('./utils/http');
 const bulkInsert = require('./utils/bulk-insert');
+const chunk = require('./utils/chunk');
 const { get } = require('./utils/qu');
 
 const pipeline = util.promisify(stream.pipeline);
@@ -63,10 +64,10 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 		? chunks.slice(0, -1).concat(chunks.slice(-1).reverse())
 		: chunks;
 
-	const groupedMedias = lastPreferredChunks.map((chunk) => {
+	const groupedMedias = lastPreferredChunks.map((mediaChunk) => {
 		// merge chunked medias into single media with grouped fallback priorities,
 		// so the first sources of each media is preferred over all second sources, etc.
-		const sources = chunk
+		const sources = mediaChunk
 			.reduce((accSources, media) => {
 				media.sources.forEach((source, index) => {
 					if (!accSources[index]) {
@@ -82,8 +83,8 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) {
 			.flat();
 
 		return {
-			id: chunk[0].id,
+			id: mediaChunk[0].id,
-			role: chunk[0].role,
+			role: mediaChunk[0].role,
 			sources,
 		};
 	});
@@ -235,22 +236,41 @@ async function findSourceDuplicates(baseMedias) {
 		.filter(Boolean);
 
 	const [existingSourceMedia, existingExtractMedia] = await Promise.all([
-		knex('media').whereIn('source', sourceUrls),
+		// my try to check thousands of URLs at once, don't pass all of them to a single query
-		knex('media').whereIn('source_page', extractUrls),
+		chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
+		chunk(extractUrls).reduce(async (chain, extractUrlsChunk) => {
+			const accUrls = await chain;
+			const existingUrls = await knex('media').whereIn('source_page', extractUrlsChunk);
+
+			return [...accUrls, ...existingUrls];
+		}, []),
 	]);
 
 	const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source');
 	const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page');
 
-	return { existingSourceMediaByUrl, existingExtractMediaByUrl };
+	return {
+		existingSourceMediaByUrl,
+		existingExtractMediaByUrl,
+	};
 }
 
 async function findHashDuplicates(medias) {
 	const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean);
 
-	const existingHashMediaEntries = await knex('media').whereIn('hash', hashes);
+	const existingHashMediaEntries = await chunk(hashes, 2).reduce(async (chain, hashesChunk) => {
-	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
+		const accHashes = await chain;
+		const existingHashes = await knex('media').whereIn('hash', hashesChunk);
 
+		return [...accHashes, ...existingHashes];
+	}, []);
+
+	const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash');
 	const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]);
 
 	const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => {
@@ -600,11 +620,11 @@ async function fetchSource(source, baseMedia) {
 			const hashStream = new stream.PassThrough();
 			let size = 0;
 
-			hashStream.on('data', (chunk) => {
+			hashStream.on('data', (streamChunk) => {
-				size += chunk.length;
+				size += streamChunk.length;
 
 				if (hasherReady) {
-					hasher.write(chunk);
+					hasher.write(streamChunk);
 				}
 			});
 

src/scrapers/dogfart.js

@@ -1,153 +1,127 @@
 'use strict';
 
-/* eslint-disable newline-per-chained-call */
-// const Promise = require('bluebird');
-const { JSDOM } = require('jsdom');
-const moment = require('moment');
-
-const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const qu = require('../utils/qu');
 
-async function getPhotos(albumUrl) {
+async function getPhotos(albumUrl, channel) {
-	const res = await http.get(albumUrl);
+	const res = await qu.get(albumUrl);
-	const html = res.body.toString();
-	const { document } = new JSDOM(html).window;
 
-	const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
+	if (!res.ok) {
+		return [];
+	}
+
+	const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1);
 	const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
 
 	const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => {
-		const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
+		const pageUrl = `${channel.url}${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
+		const networkPageUrl = `https://dogfartnetwork.com${lastPhotoPage.replace('tourx', 'tour').replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`;
 
-		return {
+		const extract = ({ query }) => query.img('.scenes-module img');
+
+		return [
+			{
 				url: pageUrl,
-			extract: ({ query }) => query.img('.scenes-module img'),
+				extract,
-		};
+			},
+			{
+				url: networkPageUrl,
+				extract,
+			},
+		];
 	});
 
 	return photoUrls;
 }
 
-function scrapeLatest(html, site, filter = true) {
+function scrapeLatest(scenes, site, filter = true) {
-	const { document } = new JSDOM(html).window;
+	return scenes.reduce((acc, { query }) => {
-	const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
+		const release = {};
 
-	return sceneElements.map((element) => {
+		const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name');
-		const siteUrl = element.querySelector('.recent-details-title .help-block, .model-details-title .site-name').textContent;
 
-		if (filter && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) {
+		release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url });
+		release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`;
+
+		release.title = query.cnt('.scene-title');
+		// release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com
+
+		// release.poster = `https:${element.querySelector('img').src}`;
+		release.poster = query.img();
+		release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url');
+
+		release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
+
+		if (filter && release.channel && `www.${release.channel}.com` !== new URL(site.url).host) {
 			// different dogfart site
-			return null;
+			return { ...acc, unextracted: [...acc.unextracted, release] };
 		}
 
-		const sceneLinkElement = element.querySelector('.thumbnail');
+		return { ...acc, scenes: [...acc.scenes, release] };
-		const url = qu.prefixUrl(sceneLinkElement.href, 'https://dogfartnetwork.com');
+	}, {
-		const { pathname } = new URL(url);
+		scenes: [],
-		const entryId = `${site.slug}_${pathname.split('/')[4]}`;
+		unextracted: [],
-
+	});
-		const title = element.querySelector('.scene-title').textContent;
-		const actors = title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim());
-
-		const poster = `https:${element.querySelector('img').src}`;
-		const teaser = sceneLinkElement.dataset.preview_clip_url;
-
-		const channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase();
-
-		return {
-			url,
-			entryId,
-			title,
-			actors,
-			poster,
-			teaser: {
-				src: teaser,
-			},
-			site,
-			channel,
-		};
-	}).filter(Boolean);
 }
 
-async function scrapeScene(html, url, site) {
+async function scrapeScene({ query }, url, channel, baseScene, parameters) {
-	const { document } = new JSDOM(html).window;
+	const release = {};
-
-	const title = document.querySelector('.description-title').textContent;
-	const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
-	const metaDescription = document.querySelector('meta[itemprop="description"]').content;
-	const description = metaDescription
-		? metaDescription.content
-		: document.querySelector('.description')
-			.textContent
-			.replace(/[ \t\n]{2,}/g, ' ')
-			.replace('...read more', '')
-			.trim();
-
-	const channel = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
 	const { origin, pathname } = new URL(url);
-	const entryId = `${channel}_${pathname.split('/').slice(-2)[0]}`;
 
-	const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
+	release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase();
-	const duration = moment
+	release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`;
-		.duration(`00:${document
-			.querySelectorAll('.extra-info p')[1]
-			.textContent
-			.match(/\d+:\d+$/)[0]}`)
-		.asSeconds();
 
-	const trailerElement = document.querySelector('.html5-video');
+	release.title = query.cnt('.description-title') || query.text('.scene-title');
-	const poster = `https:${trailerElement.dataset.poster}`;
+	release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({
-	const { trailer } = trailerElement.dataset;
+		name: query.cnt(actorEl),
+		url: query.url(actorEl, null, 'href', { origin: channel.url }),
+	}));
 
-	const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0]?.href;
+	release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description, [itemprop="description"]')?.replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim();
-	const photos = lastPhotosUrl ? await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url) : [];
 
-	const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
+	release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content');
-	const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
+	release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container');
 
-	return {
+	release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/);
-		entryId,
+
-		url: `${origin}${pathname}`,
+	const trailer = query.video('.html5-video', 'data-trailer');
-		title,
+	const lastPhotosUrl = query.urls('.pagination a').at(-1);
-		description,
+
-		actors,
+	release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image');
-		date,
+
-		duration,
+	if (trailer && !trailer?.includes('join')) {
-		poster,
+		release.trailer = trailer;
-		photos,
+	}
-		trailer: {
+
-			src: trailer,
+	if (lastPhotosUrl && parameters.includePhotos) {
-		},
+		release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url);
-		tags,
+	}
-		rating: {
+
-			stars,
+	release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2));
-		},
+
-		site,
+	return release;
-		channel,
-	};
 }
 
-async function fetchLatest(site, page = 1) {
+async function fetchLatest(channel, page = 1, { parameters }) {
-	const res = await http.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+	const res = await qu.getAll(parameters.latest ? `${parameters.latest}/?p=${page}` : `${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container');
 
-	return scrapeLatest(res.body.toString(), site);
+	if (res.ok) {
-}
+		return scrapeLatest(res.items, channel);
+	}
 
-async function fetchScene(url, site) {
+	return res.status;
-	const res = await http.get(url);
-
-	return scrapeScene(res.body.toString(), url, site);
 }
 
 async function fetchProfile(baseActor, entity) {
 	const slug = slugify(baseActor.name, '+');
 	const url = `https://www.dogfartnetwork.com/tour/girls/${slug}/`;
 
-	const res = await http.get(url);
+	const res = await qu.getAll(url, '.recent-updates');
 
 	if (res.ok) {
-		const scenes = scrapeLatest(res.body, entity, false);
+		const { scenes } = scrapeLatest(res.items, entity, false);
 
+		// no bio available
 		return { scenes };
 	}
 
@@ -156,6 +130,6 @@ async function fetchProfile(baseActor, entity) {
 
 module.exports = {
 	fetchLatest,
-	fetchScene,
 	fetchProfile,
+	scrapeScene,
 };

src/scrapers/gamma.js

@@ -457,7 +457,7 @@ async function scrapeReleaseApi(data, site, options) {
 		release.trailer = Object.entries(data.trailers).map(([quality, source]) => ({ src: source, quality }));
 	}
 
-	if (data.movie_id && !data.movie_path) {
+	if (data.movie_id && !data.movie_path && options.parameters.sceneMovies !== false) {
 		release.movie = {
 			entryId: data.movie_id,
 			title: data.movie_title,

src/scrapers/insex.js

@@ -5,6 +5,27 @@ const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
 function scrapeLatest(scenes, site) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+
+		release.title = query.cnt('.has-text-weight-bold, .is-size-6');
+		release.date = query.date('span.tag', 'YYYY-MM-DD');
+		release.actors = query.cnts('a.tag');
+
+		const cover = query.img('.image img');
+
+		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
+		release.covers = [cover];
+
+		release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
+
+		return release;
+	});
+}
+
+function scrapeLatestLegacy(scenes, site) {
 	return scenes.map(({ query }) => {
 		// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
 		const release = {};
@@ -47,28 +68,35 @@ function scrapeLatest(scenes, site) {
 	});
 }
 
-function scrapeLatestAlt(scenes, site) {
+async function scrapeScene({ query }, url, channel, parameters, session) {
-	return scenes.map(({ query }) => {
 	const release = {};
 
-		release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
+	release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
+	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
+	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
 
-		release.title = query.cnt('.has-text-weight-bold');
+	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
-		release.date = query.date('span.tag', 'YYYY-MM-DD');
+	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
-		release.actors = query.cnts('a.tag');
 
-		const cover = query.img('.image img');
+	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
-
+	release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
-		release.poster = cover.replace('poster_noplay', 'trailer_noplay');
-		release.covers = [cover];
 
 	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
 
+	release.trailer = query.video();
+
+	if (!release.trailer && parameters.includeTrailers) {
+		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
+
+		if (trailerRes.ok) {
+			release.trailer = trailerRes.body;
+		}
+	}
+
 	return release;
-	});
 }
 
-function scrapeScene({ query }, site) {
+function scrapeSceneLegacy({ query }, site) {
 	const release = {};
 
 	const titleEl = query.q('.articleTitleText');
@@ -97,70 +125,34 @@ function scrapeScene({ query }, site) {
 	return release;
 }
 
-async function scrapeSceneAlt({ query }, url, channel, session) {
-	const release = {};
-
-	release.title = query.cnt('.columns div.is-size-5');
-	release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
-	release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
-
-	release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
-	release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
-
-	release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
-	release.photos = query.imgs('body > div:nth-child(6) img');
-
-	release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
-
-	release.trailer = query.video();
-
-	if (!release.trailer) {
-		const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
-
-		if (trailerRes.ok) {
-			release.trailer = trailerRes.body;
-		}
-	}
-
-	return release;
-}
-
 async function fetchLatest(site, page = 1) {
-	const url = (site.parameters?.scraper === 'alt' && `${site.parameters.latest}/home.php?o=latest&p=${page}`)
+	const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
-		// || (site.slug === 'paintoy' && `${site.url}/corporal/punishment/gallery.php?type=brief&page=${page}`) // paintoy's site is (was?) partially broken, use front page
+	const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
-		|| `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-
-	const res = await ((site.parameters?.scraper === 'alt' && qu.getAll(url, 'body > .columns .column'))
-		// || (site.slug === 'paintoy' && qu.getAll(url, '#articleTable table[cellspacing="2"]'))
-		|| qu.get(url)); // JSON containing html as a property
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
-			return scrapeLatestAlt(res.items, site);
-		}
-
-		/*
-		if (site.slug === 'paintoy') {
 		return scrapeLatest(res.items, site);
 	}
-		*/
-
-		return scrapeLatest(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
-	}
 
 	return res.status;
 }
 
-async function fetchScene(url, site) {
+async function fetchLatestLegacy(site, page = 1) {
-	const session = http.session();
+	const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
-	const res = await qu.get(url, null, null, { session });
+	const res = await qu.get(url); // JSON containing html as a property
 
 	if (res.ok) {
-		if (site.parameters?.scraper === 'alt') {
+		return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
-			return scrapeSceneAlt(res.item, url, site, session);
 	}
 
-		return scrapeScene(res.item, site);
+	return res.status;
+}
+
+async function fetchScene(url, site, baseRelease, parameters) {
+	const session = http.session();
+	const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
+
+	if (res.ok) {
+		return scrapeScene(res.item, url, site, parameters, session);
 	}
 
 	return res.status;
@@ -169,4 +161,8 @@ async function fetchScene(url, site) {
 module.exports = {
 	fetchLatest,
 	fetchScene,
+	legacy: {
+		fetchLatest: fetchLatestLegacy,
+		scrapeScene: scrapeSceneLegacy,
+	},
 };

src/scrapers/kellymadison.js

@@ -16,7 +16,7 @@ const siteMapByKey = {
 const siteMapBySlug = Object.entries(siteMapByKey).reduce((acc, [key, value]) => ({ ...acc, [value]: key }), {});
 
 function scrapeLatest(scenes, site) {
-	return scenes.map(({ query }) => {
+	return scenes.reduce((acc, { query }) => {
 		const release = {};
 
 		release.shootId = query.q('.card-meta .text-right, .row .text-right, .card-footer-item:last-child', true);
@@ -24,11 +24,6 @@ function scrapeLatest(scenes, site) {
 		const siteId = release.shootId.match(/\d?\w{2}/)[0];
 		const siteSlug = siteMapByKey[siteId];
 
-		if (site.slug !== siteSlug) {
-			// using generic network overview, scene is not from the site we want
-			return null;
-		}
-
 		const { pathname } = new URL(query.url('h5 a, .ep-title a, .title a'));
 		[release.entryId] = pathname.match(/\d+$/);
 		release.url = `${site.url}${pathname}`;
@@ -52,8 +47,16 @@ function scrapeLatest(scenes, site) {
 			};
 		}
 
-		return release;
+		if (site.slug !== siteSlug) {
-	}).filter((scene) => scene);
+			// using generic network overview, scene is not from the site we want
+			return { ...acc, unextracted: [...acc.unextracted, release] };
+		}
+
+		return { ...acc, scenes: [...acc.scenes, release] };
+	}, {
+		scenes: [],
+		unextracted: [],
+	});
 }
 
 async function scrapeScene({ query, html }, url, baseRelease, channel, session) {

src/scrapers/kink.js

@@ -131,7 +131,18 @@ async function scrapeProfile({ query }, actorUrl, include) {
 }
 
 async function fetchLatest(site, page = 1) {
-	const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
+	// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
+	const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
+		Host: 'www.kink.com',
+		'User-Agent': 'HTTPie/2.6.0',
+		'Accept-Encoding': 'gzip, deflate, br',
+		Accept: '*/*',
+		Connection: 'keep-alive',
+
+	}, {
+		includeDefaultHeaders: false,
+		followRedirects: false,
+	});
 
 	if (res.ok) {
 		return scrapeAll(res.items, site);

src/scrapers/mindgeek.js

@@ -242,29 +242,22 @@ async function getSession(site, parameters, url) {
 	throw new Error(`Failed to acquire MindGeek session (${res.statusCode})`);
 }
 
-function scrapeProfile(data, html, releases = [], networkName) {
+function scrapeProfile(data, releases = [], networkName) {
-	const { query } = qu.extract(html);
-
 	const profile = {
 		description: data.bio,
-		aliases: data.aliases,
+		aliases: data.aliases.filter(Boolean),
 	};
 
 	profile.gender = data.gender === 'other' ? 'transsexual' : data.gender;
+	profile.measurements = data.measurements;
 
-	if (data.measurements) {
+	profile.dateOfBirth = qu.parseDate(data.birthday);
-		const [bust, waist, hip] = data.measurements.split('-');
+	profile.birthPlace = data.birthPlace;
+	profile.height = inchesToCm(data.height);
+	profile.weight = lbsToKg(data.weight);
 
-		if (profile.gender === 'female') {
+	profile.hairColor = data.tags.find((tag) => /hair color/i.test(tag.category))?.name;
-			if (bust) profile.bust = bust.toUpperCase();
+	profile.ethnicity = data.tags.find((tag) => /ethnicity/i.test(tag.category))?.name;
-			if (waist) profile.waist = waist;
-			if (hip) profile.hip = hip;
-		}
-	}
-
-	if (data.birthPlace) profile.birthPlace = data.birthPlace;
-	if (data.height) profile.height = inchesToCm(data.height);
-	if (data.weight) profile.weight = lbsToKg(data.weight);
 
 	if (data.images.card_main_rect?.[0]) {
 		profile.avatar = data.images.card_main_rect[0].xl?.url
@@ -274,9 +267,6 @@ function scrapeProfile(data, html, releases = [], networkName) {
             || data.images.card_main_rect[0].xs?.url;
 	}
 
-	const birthdate = query.all('li').find((el) => /Date of Birth/.test(el.textContent));
-	if (birthdate) profile.birthdate = query.date(birthdate, 'span', 'MMMM Do, YYYY');
-
 	if (data.tags.some((tag) => /boob type/i.test(tag.category) && /natural tits/i.test(tag.name))) {
 		profile.naturalBoobs = true;
 	}
@@ -285,6 +275,14 @@ function scrapeProfile(data, html, releases = [], networkName) {
 		profile.naturalBoobs = false;
 	}
 
+	if (data.tags.some((tag) => /body art/i.test(tag.category) && /tattoo/i.test(tag.name))) {
+		profile.hasTattoos = true;
+	}
+
+	if (data.tags.some((tag) => /body art/i.test(tag.category) && /piercing/i.test(tag.name))) {
+		profile.hasPiercings = true;
+	}
+
 	profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName));
 
 	return profile;
@@ -377,7 +375,7 @@ async function fetchRelease(url, site, baseScene, options) {
 	return null;
 }
 
-async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, parameters }) {
+async function fetchProfile({ name: actorName }, { entity, parameters }, include) {
 	// const url = `https://www.${networkOrNetworkSlug.slug || networkOrNetworkSlug}.com`;
 	const { session, instanceToken } = await getSession(entity, parameters);
 
@@ -395,31 +393,22 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
 		const actorData = res.body.result.find((actor) => actor.name.toLowerCase() === actorName.toLowerCase());
 
 		if (actorData) {
-			const actorUrl = `https://www.${entity.slug}.com/${entity.parameters?.actorPath || 'model'}/${actorData.id}/${actorSlug}`;
 			const actorReleasesUrl = `https://site-api.project1service.com/v2/releases?actorId=${actorData.id}&limit=100&offset=0&orderBy=-dateReleased&type=scene`;
 
-			const [actorRes, actorReleasesRes] = await Promise.all([
+			const actorReleasesRes = include.includeActorScenes && await http.get(actorReleasesUrl, {
-				http.get(actorUrl, {
-					interval: parameters.interval,
-					concurrency: parameters.concurrency,
-				}),
-				http.get(actorReleasesUrl, {
 				session,
 				interval: parameters.interval,
 				concurrency: parameters.concurrency,
 				headers: {
 					Instance: instanceToken,
 				},
-				}),
+			});
-			]);
 
-			if (actorRes.statusCode === 200 && actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
+			if (actorReleasesRes.statusCode === 200 && actorReleasesRes.body.result) {
-				return scrapeProfile(actorData, actorRes.body.toString(), actorReleasesRes.body.result, entity.slug);
+				return scrapeProfile(actorData, actorReleasesRes.body.result, entity.slug);
 			}
 
-			if (actorRes.statusCode === 200) {
+			return scrapeProfile(actorData, [], entity.slug);
-				return scrapeProfile(actorData, actorRes.body.toString(), null, entity.slug);
-			}
 		}
 	}
 

src/scrapers/pervcity.js

@@ -12,6 +12,13 @@ const channelCodes = {
 	uha: 'upherasshole',
 };
 
+const qualities = {
+	v4k: 2160,
+	vFullHD: 1080,
+	vHD: 720,
+	vSD: 480,
+};
+
 const channelRegExp = new RegExp(Object.keys(channelCodes).join('|'), 'i');
 
 function scrapeAll(scenes, entity) {
@@ -36,15 +43,18 @@ function scrapeAll(scenes, entity) {
 	});
 }
 
-function scrapeScene({ query }) {
+function scrapeScene({ query }, channel) {
 	const release = {};
 
 	release.entryId = query.q('.trailerLeft img', 'id').match(/set-target-(\d+)/)[1];
 
 	release.title = query.cnt('.infoHeader h1');
-	release.description = query.cnt('.infoBox p');
+	release.description = query.cnt('.description');
+	release.duration = query.duration('.tRuntime');
 
 	release.actors = query.cnts('.infoBox .tour_update_models a');
+	release.tags = query.cnts('.tagcats a');
+	release.qualities = query.imgs('.avaiFormate img').map((src) => qualities[src.match(/\/(\w+)\.png/)[1]]).filter(Boolean);
 
 	release.poster = query.img('.posterimg');
 	release.photos = query.imgs('.trailerSnaps img').slice(1); // first photo is poster in lower quality
@@ -52,7 +62,7 @@ function scrapeScene({ query }) {
 	const trailer = query.q('script')?.textContent.match(/\/trailers\/.+\.mp4/)?.[0];
 
 	if (trailer) {
-		release.trailer = `https://pervcity.com${trailer}`;
+		release.trailer = `${channel.url}${trailer}`;
 		release.channel = channelCodes[release.trailer.match(channelRegExp)?.[0]];
 	}
 
@@ -85,22 +95,34 @@ function scrapeProfile({ query }) {
 	return profile;
 }
 
-async function fetchLatest(channel, page = 1) {
+function getLatestUrl(channel, page) {
 	if (channel.parameters?.siteId) {
-		const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
+		return `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
-		const res = await qu.getAll(url, '.videoBlock');
+	}
 
-		return res.ok ? scrapeAll(res.items, channel) : res.status;
+	if (channel.parameters?.native) {
+		return `${channel.url}/search.php?site[]=&page=${page}`;
 	}
 
 	return null;
 }
 
-async function fetchUpcoming(channel) {
+async function fetchLatest(channel, page = 1) {
-	const url = 'https://pervcity.com';
+	const url = getLatestUrl(channel, page);
-	const res = await qu.getAll(url, '.upcoming .videoBlock');
 
-	return res.ok ? scrapeAll(res.items, channel.parent) : res.status;
+	if (url) {
+		const res = await qu.getAll(url, '.videoBlock');
+
+		return res.ok ? scrapeAll(res.items, channel) : res.status;
+	}
+
+	return [];
+}
+
+async function fetchUpcoming(channel) {
+	const res = await qu.getAll(channel.url, '.upcoming .videoBlock');
+
+	return res.ok ? scrapeAll(res.items, channel.parameters?.native ? channel : channel.parent) : res.status;
 }
 
 async function fetchScene(url, entity) {

src/scrapers/rickysroom.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const qu = require('../utils/q');
+const { lbsToKg, feetInchesToCm } = require('../utils/convert');
+
+function scrapeScene(data, channel) {
+	const release = {};
+
+	release.entryId = data.id;
+	release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
+
+	release.title = data.title;
+	release.description = data.description;
+	release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
+	release.duration = qu.durationToSeconds(data.videos_duration);
+
+	release.actors = data?.models_thumbs.map((model) => ({
+		name: model.name,
+		url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
+		avatar: model.thumb,
+	})) || data.models;
+
+	release.tags = data.tags;
+
+	release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
+	release.photos = data.previews.full
+		.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
+		.concat(data.thumbs); // screenshots
+
+	release.trailer = data.trailer_url;
+	release.teaser = data.special_thumbnails;
+
+	release.qualities = data.videos && Object.values(data.videos).map((video) => video.height);
+	release.rating = data.rating;
+
+	return release;
+}
+
+function scrapeProfile(data, scenes, entity) {
+	const profile = {};
+
+	profile.entryId = data.id;
+	profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
+
+	profile.description = data.Bio || data.bio;
+	profile.birthPlace = data.Born || data.born;
+	profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
+
+	profile.measurements = data.Measurements || data.Measurements;
+	profile.height = feetInchesToCm(data.Height || data.height);
+	profile.weight = lbsToKg(data.Weight || data.weight);
+
+	profile.eyes = data.Eyes || data.eyes;
+	profile.hairColor = data.Hair || data.hair;
+
+	profile.avatar = data.thumb;
+
+	profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
+
+	return profile;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.contents?.data) {
+			return data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+async function fetchScene(url, channel, baseRelease) {
+	if (baseRelease.entryId) {
+		// deep data is identical to update data
+		return baseRelease;
+	}
+
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.content) {
+			return scrapeScene(data.props.pageProps.content, channel);
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+async function fetchProfile({ slug }, entity) {
+	const url = `${entity.url}/models/${slug}`;
+	const res = await qu.get(url);
+
+	if (res.ok) {
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data.props?.pageProps?.model) {
+			return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
+		}
+
+		return null;
+	}
+
+	return res.status;
+}
+
+module.exports = {
+	fetchLatest,
+	fetchScene,
+	fetchProfile,
+};