Refactored Vixen scraper, moved XEmpire to generic Gamma scraper after site update, added BiPhoria.

assets/components/container/container.vue

@@ -107,6 +107,7 @@ export default {
 			showSidebar: false,
 			showWarning: localStorage.getItem('consent') !== window.env.sessionId,
 			showFilters: false,
+			selected: null,
 		};
 	},
 	mounted,

public/img/logos/biphoria/misc/biphoria-light.svg

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 8192 696.7" style="enable-background:new 0 0 8192 696.7;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+</style>
+<g>
+	<path class="st0" d="M378.2,11.5c103,0,155.7,10.1,203.2,33.6c47.5,24.4,74,62.1,74,116c0,87.5-109.6,134.3-187.4,145.5v2
+		c134.6,12.2,237.6,68.2,237.6,170c0,72.2-46.2,120.1-118.8,153.7C524.8,660.8,431.1,673,334.7,673H6v-28.5
+		c103-6.1,113.6-14.2,113.6-124.1V164.2C119.6,54.2,109,46.1,13.9,40V11.5L378.2,11.5L378.2,11.5z M264.7,300.6h52.8
+		c113.6,0,184.8-40.7,184.8-126.2c0-93.6-85.8-124.1-166.3-124.1c-34.4,0-52.8,2-60.7,5.1c-10.6,4.1-10.6,21.4-10.6,45.7
+		L264.7,300.6L264.7,300.6z M264.7,337.2v181.1c0,95.6,15.8,116,100.3,114c87.1,0,176.9-37.6,176.9-142.5
+		c0-103.8-97.7-152.6-232.4-152.6L264.7,337.2L264.7,337.2z"/>
+	<path class="st0" d="M1438.2,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L1438.2,561L1438.2,561z"/>
+	<path class="st0" d="M2380.7,11.5c99,0,174.2,12.2,227,40.7c56.7,30.5,89.8,75.3,89.8,143.5c0,129.2-133.4,195.3-261.4,207.6
+		c-21.1,2-44.8,2-60.7,2l-88.4-18.4v133.3c0,109.9,10.6,118.1,132,124.1v28.5h-388.1v-28.4c100.3-6.1,110.9-14.2,110.9-124.1V164.2
+		c0-109.9-10.6-118.1-105.6-124.1V11.5L2380.7,11.5L2380.7,11.5z M2287,347.3c21.1,6.1,46.2,12.2,80.5,12.2
+		c55.4,0,169-28.5,169-162.8c0-104.8-74-146.6-184.8-146.6c-63.4,0-64.6,6.1-64.6,36.6V347.3z"/>
+	<path class="st0" d="M3690.2,412.4V331c0-75.3-7.9-79.4-93.8-83.5V219h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v229.9
+		c0,74.2,7.9,79.4,89.8,83.5v28.5h-322.1v-28.4c85.8-4.1,93.8-9.1,93.8-83.5V453.1h-269.3V561c0,74.2,7.9,79.4,85.8,83.5V673h-315.5
+		v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1c0-75.3-7.9-79.4-93.8-83.5v-28.5h322.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5v81.4
+		L3690.2,412.4L3690.2,412.4z"/>
+	<path class="st0" d="M5085.4,438.9c0,160.8-159.7,246.2-327.4,246.2c-219.1,0-330-120.1-330-228.9c0-168.9,175.5-249.3,330-249.3
+		C4958.7,207,5085.4,310.7,5085.4,438.9z M4767.3,648.5c84.4,0,162.4-51.9,162.4-187.2c0-108.9-64.6-217.8-183.5-217.8
+		c-87.1,0-162.4,70.2-162.4,189.3C4583.8,557,4656.4,648.5,4767.3,648.5z"/>
+	<path class="st0" d="M5816.7,561c0,74.2,7.9,79.4,89.8,83.5V673h-314.1v-28.5c83.2-4.1,91.1-9.1,91.1-83.5V331.1
+		c0-75.3-7.9-79.4-87.1-83.5v-28.5h330c60.7,0,110.9,6.1,149.2,23.4c36.9,17.3,66,50.9,66,95.6c0,60-55.4,97.7-128,116
+		c15.8,21.4,58.1,70.2,85.8,100.7c31.7,35.6,56.7,57,72.6,68.2c18.5,12.2,43.6,23.4,66,29.5l-4,26.5h-52.8
+		c-117.5-1-153.2-26.5-188.8-63.1c-31.7-32.6-66-82.5-88.4-111c-15.8-21.4-26.4-25.5-66-25.5h-21.3V561z M5816.7,447h42.3
+		c29,0,62.1-3,85.8-14.2c36.9-17.3,52.8-47.9,52.8-83.5c0-65.1-58.1-94.6-124-94.6c-54.2,0-56.7,1-56.7,29.5L5816.7,447L5816.7,447z
+		"/>
+	<path class="st0" d="M6948,561c0,74.2,7.9,79.4,89.8,83.5V673h-318.1v-28.5c81.9-4.1,89.8-9.1,89.8-83.5V331.1
+		c0-75.3-7.9-79.4-89.8-83.5v-28.5h318.1v28.5c-81.9,4.1-89.8,8.1-89.8,83.5L6948,561L6948,561z"/>
+	<path class="st0" d="M7886.5,644.5l25-2c36.9-3,43.6-10.1,21.1-54l-31.7-61.1h-180.9c-5.2,11.2-19.8,43.7-31.7,73.2
+		c-11.9,30.5-4,38.6,33,41.7l30.4,2v28.5h-238.9v-28.4c55.4-5.1,79.2-7.1,116.1-74.2l196.7-355.2l48.8-8.1l199.3,375.5
+		c30.4,56,56.7,58,112.2,62.1V673h-299.7v-28.5H7886.5z M7734.7,488.8h147.8l-72.6-144.5h-4L7734.7,488.8z"/>
+</g>
+</svg>

seeds/01_networks.js

@@ -627,7 +627,9 @@ const networks = [
 		url: 'https://www.xempire.com',
 		description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
 		parameters: {
+			layout: 'api',
 			actorScenes: 'https://www.xempire.com/en/videos/xempire/latest/{page}/All-Categories/0{actorPath}',
+			sceneMovies: false,
 		},
 		parent: 'gamma',
 	},

seeds/02_sites.js

@@ -1685,6 +1685,18 @@ const sites = [
 			layout: 'members',
 		},
 	},
+	// BIPHORIA
+	{
+		slug: 'biphoria',
+		name: 'BiPhoria',
+		url: 'https://www.biphoria.com',
+		independent: true,
+		tags: ['bisexual'],
+		parameters: {
+			layout: 'api',
+		},
+		parent: 'gamma',
+	},
 	// BLOWPASS
 	{
 		slug: '1000facials',
@@ -11069,6 +11081,7 @@ const sites = [
 		description: 'Watch Lesbian porn videos with the highest quality all girl on girl sex videos featuring SLAYED pornstars and models. Only the highest quality lesbian sex videos exclusive to SLAYED.com',
 		url: 'https://www.slayed.com',
 		parent: 'vixen',
+		tags: ['lesbian'],
 	},
 	// VOGOV
 	{

seeds/04_media.js

@@ -600,6 +600,7 @@ const tagMedia = [
 	['airtight', 6, 'Remy Lacroix in "Ass Worship 14"', 'julesjordan'],
 	['airtight', 'anissa_kate_legalporno', 'Anissa Kate in GP1962', 'analvids'],
 	['airtight', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
+	['airtight', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['airtight', 'diamond_foxxx_milfslikeitbig', 'Diamond Foxx in "Diamond\'s Bday Gangbang"', 'milfslikeitbig'],
 	['airtight', 'tory_lane_bigtitsatwork', 'Tory Lane in "I\'m Your Christmas Bonus"', 'bigtitsatwork'],
 	['airtight', 11, 'Malena Nazionale in "Rocco\'s Perverted Secretaries 2: Italian Edition"', 'roccosiffredi'],
@@ -904,6 +905,7 @@ const tagMedia = [
 	['free-use', 'veruca_james_brazzersexxtra', 'Veruca James in "The Perfect Maid"', 'brazzersexxtra'],
 	['free-use', 'gia_dibella_freeusefantasy', 'Gia Dibella in "Learning to Freeuse"', 'freeusefantasy'],
 	['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9"', 'julesjordan'],
+	['gangbang', 'savannah_bond_julesjordan', 'Savannah Bond', 'julesjordan'],
 	['gangbang', 'kristen_scott_julesjordan', 'Kristen Scott in "Interracial Gangbang!"', 'julesjordan'],
 	['gangbang', 'emily_willis_blacked', 'Emily Willis', 'blacked'],
 	['gangbang', 'monika_fox_legalporno', 'Monika Fox in GL479', 'analvids'],

src/app.js

@@ -195,6 +195,7 @@ async function init() {
 			await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
 		}
 	} catch (error) {
+		console.trace(error);
 		logger.error(error);
 	}
 

src/scrapers/gamma.js

@@ -457,7 +457,7 @@ async function scrapeReleaseApi(data, site, options) {
 		release.trailer = Object.entries(data.trailers).map(([quality, source]) => ({ src: source, quality }));
 	}
 
-	if (data.movie_id && !data.movie_path) {
+	if (data.movie_id && !data.movie_path && options.parameters.sceneMovies !== false) {
 		release.movie = {
 			entryId: data.movie_id,
 			title: data.movie_title,

src/scrapers/kink.js

@@ -131,7 +131,18 @@ async function scrapeProfile({ query }, actorUrl, include) {
 }
 
 async function fetchLatest(site, page = 1) {
-	const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
+	// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
+	const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
+		Host: 'www.kink.com',
+		'User-Agent': 'HTTPie/2.6.0',
+		'Accept-Encoding': 'gzip, deflate, br',
+		Accept: '*/*',
+		Connection: 'keep-alive',
+
+	}, {
+		includeDefaultHeaders: false,
+		followRedirects: false,
+	});
 
 	if (res.ok) {
 		return scrapeAll(res.items, site);

src/scrapers/scrapers.js

@@ -65,7 +65,7 @@ const vixen = require('./vixen');
 const vogov = require('./vogov');
 const wankzvr = require('./wankzvr');
 const whalemember = require('./whalemember');
-const xempire = require('./xempire');
+// const xempire = require('./xempire');
 
 // profiles
 const boobpedia = require('./boobpedia');
@@ -157,7 +157,7 @@ const scrapers = {
 		wankzvr,
 		westcoastproductions: adultempire,
 		whalemember,
-		xempire,
+		// xempire,
 	},
 	actors: {
 		'18vr': badoink,
@@ -291,7 +291,7 @@ const scrapers = {
 		westcoastproductions: adultempire,
 		wicked: gamma,
 		wildoncam: cherrypimps,
-		xempire,
+		xempire: gamma,
 	},
 };
 

src/scrapers/vixen.js

@@ -4,7 +4,7 @@
 const Promise = require('bluebird');
 const moment = require('moment');
 
-const logger = require('../logger')(__filename);
+const qu = require('../utils/qu');
 const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 
@@ -49,6 +49,26 @@ function getAvatarFallbacks(avatar) {
 		.flat();
 }
 
+function curateSources(sources, type = 'image/jpeg') {
+	if (!sources) {
+		return null;
+	}
+
+	return sources
+		.map((source) => ({
+			src: source.src,
+			width: source.width,
+			height: source.height,
+			type: source.type || type,
+			expectType: {
+				'binary/octet-stream': type,
+			},
+		}))
+		.sort((resA, resB) => (resB.width * resB.height) - (resA.width * resA.height))			// number of pixels
+		.sort((resA, resB) => Math.abs(1.8 - Number((resA.width / resA.height).toFixed(1)))		// approximation to 16:9
+							- Math.abs(1.8 - Number((resB.width / resB.height).toFixed(1))));
+}
+
 async function getTrailer(scene, channel, url) {
 	const res = await http.post(`${channel.url}/graphql`, {
 		operationName: 'getToken',
@@ -142,88 +162,27 @@ async function getTrailer(scene, channel, url) {
 	return null;
 }
 
-/*
-async function getPhotosLegacy(url) {
-	const htmlRes = await http.get(url, {
-		extract: {
-			runScripts: 'dangerously',
-		},
-	});
-
-	try {
-		const state = htmlRes?.window?.__APOLLO_STATE__;
-
-		if (!state) {
-			return [];
-		}
-
-		const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
-		const data = state[key];
-
-		if (!data) {
-			return [];
-		}
-
-		return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
-	} catch (error) {
-		logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
-		return [];
-	}
-}
-*/
-
-async function getPhotos(url) {
-	const htmlRes = await http.get(url, {
-		parse: true,
-		extract: {
-			runScripts: 'dangerously',
-		},
-	});
-
-	try {
-		const state = htmlRes?.window?.__APOLLO_STATE__;
-
-		console.log('state', state);
-
-		if (!state) {
-			return [];
-		}
-
-		const key = Object.values(state?.ROOT_QUERY).find((query) => query?.__ref)?.__ref;
-		const data = state[key];
-
-		console.log('data', data);
-
-		if (!data) {
-			return [];
-		}
-
-		console.log(data.carousel);
-
-		return data.carousel.slice(1).map((photo) => photo.main?.[0].src).filter(Boolean);
-	} catch (error) {
-		logger.warn(`Failed to retrieve Vixen images: ${error.message}`);
-		return [];
-	}
-}
-
-function scrapeAll(scenes, site, origin) {
-	return scenes.map((scene) => {
+function scrapeAll(scenes, channel) {
+	return scenes.map((data) => {
 		const release = {};
 
-		release.title = scene.title;
+		release.entryId = data.videoId;
+		release.url = `${channel.url}/videos/${data.slug}`;
+		release.title = data.title;
 
-		release.entryId = String(scene.newId);
-		release.url = `${site?.url || origin}/videos${scene.targetUrl}`;
+		release.date = qu.extractDate(data.releaseDate);
+		release.actors = data.modelsSlugged.map((model) => ({
+			name: model.name,
+			url: `${channel.url}/models/${model.slugged}`,
+		}));
 
-		release.date = moment.utc(scene.releaseDate).toDate();
-		release.datePrecision = 'minute';
+		release.poster = curateSources(data.images.listing);
+		release.teaser = curateSources(data.previews.listing, 'video/mp4');
 
-		release.actors = scene.models;
-		release.stars = Number(scene.textRating) / 2;
+		release.stars = data.rating;
 
-		release.poster = getPosterFallbacks(scene.images.poster);
-		release.teaser = getTeaserFallbacks(scene.previews.poster);
+		console.log(data);
+		console.log(release);
 
 		return release;
 	});
@@ -252,47 +211,47 @@ function scrapeUpcoming(scene, site) {
 
 	release.entryId = (release.poster[0] || release.teaser[0])?.src?.match(/\/(\d+)/)?.[1];
 
+	console.log('upcoming', scene);
+
 	return [release];
 }
 
-async function scrapeScene(data, url, site, baseRelease, options) {
-	const scene = data.video;
-
+async function scrapeScene(data, url, channel, options) {
 	const release = {
 		url,
-		title: scene.title,
-		description: scene.description,
-		actors: scene.models,
-		director: scene.directorNames,
-		duration: scene.runLength,
-		stars: scene.totalRateVal,
-		tags: scene.tags,
+		entryId: data.video.videoId || data.video.newId,
+		title: data.video.title,
+		description: data.video.description,
+		actors: data.video.models,
+		director: data.video.directorNames,
+		duration: qu.durationToSeconds(data.video.runLength),
+		stars: data.video.rating,
 	};
 
-	release.entryId = scene.newId;
+	release.entryId = data.video.newId;
+	release.date = qu.extractDate(data.video.releaseDate);
 
-	release.date = moment.utc(scene.releaseDate).toDate();
-	release.productionDate = moment.utc(scene.shootDate).toDate();
-	release.datePrecision = 'minute';
+	release.actors = data.video.modelsSlugged.map((model) => ({
+		name: model.name,
+		url: `${channel.url}/models/${model.slugged}`,
+	}));
 
-	release.actors = baseRelease?.actors || scene.models;
+	release.poster = curateSources(data.video.images?.poster) || data.video.videoImage?.src;
+	release.photos = data.galleryImages?.length > 0
+		? data.galleryImages.map((image) => image.src)
+		: data.video.carousel?.map((photo) => photo.main[0]?.src).filter(Boolean);
 
-	release.poster = getPosterFallbacks(scene.images.poster);
+	if (options.includeTrailers) {
+		const trailer = await getTrailer(data.video, channel, url);
 
-	// release.photos = data.pictureset.map(photo => photo.main[0]?.src).filter(Boolean);
-	if (options.includePhotos) {
-		release.photos = await getPhotos(url);
+		if (trailer) {
+			release.trailer = trailer;
+		}
 	}
 
-	release.teaser = getTeaserFallbacks(scene.previews.poster);
+	release.qualities = data.video?.downloadResolutions.map((quality) => Number(quality.width)).filter(Boolean); // width property is actually the height
 
-	const trailer = await getTrailer(scene, site, url);
-	if (trailer) release.trailer = trailer;
-
-	release.chapters = data.video.chapters?.video.map((chapter) => ({
-		tags: [chapter.title],
-		time: chapter.seconds,
-	}));
+	console.log(release);
 
 	return release;
 }
@@ -346,13 +305,71 @@ async function scrapeProfile(data, origin, withReleases) {
 	return profile;
 }
 
+async function fetchLatestGraphql(channel, page = 1) {
+	const query = `
+		query($query: String!, $site: Site!) {
+		  searchVideos(input: {
+			  query: $query
+			  site: $site
+		  }) {
+			edges {
+			  node {
+				title
+				slug
+				description
+				releaseDate
+				categories {
+				  name
+				}
+				chapters {
+				  video {
+					title
+					seconds
+				  }
+				}
+				models {
+				  name
+				}
+				images {
+				  poster {
+					...ImageInfo
+				  }
+				}
+			  }
+			}
+		  }
+		}
+
+		fragment ImageInfo on Image {
+		  src
+		  highdpi {
+			double
+		  }
+		}
+	`;
+
+	const variables = JSON.stringify({
+		site: channel.slug.toUpperCase(),
+		query: 'alone at last',
+	});
+
+	const res = await http.get(`${channel.url}/graphql?query=${encodeURI(query)}&variables=${variables}`);
+
+	console.log(res.body);
+	console.log(res.body.errors);
+	console.log(res.body.data?.searchVideos?.edges.map((edge) => edge.node));
+}
+
 async function fetchLatest(site, page = 1) {
-	const url = `${site.url}/api/videos?page=${page}`;
-	const res = await http.get(url);
+	const url = `${site.url}/videos?page=${page}`;
+	const res = await qu.get(url);
 
 	if (res.ok) {
-		if (res.body.data.videos) {
-			return scrapeAll(res.body.data.videos, site);
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
+
+		if (data?.props.pageProps.edges) {
+			return scrapeAll(data.props.pageProps.edges.map((edge) => edge.node), site);
 		}
 
 		return [];
@@ -376,22 +393,14 @@ async function fetchUpcoming(site) {
 	return res.status;
 }
 
-async function fetchScene(url, site, baseRelease, options) {
-	const { origin, pathname } = new URL(url);
-	const apiUrl = `${origin}/api/${pathname.split('/').slice(-1)[0]}`;
-
-	const res = await http.get(apiUrl, {
-		extract: {
-			runScripts: 'dangerously',
-		},
-	});
+async function fetchScene(url, channel, baseRelease, options) {
+	const res = await qu.get(url);
 
 	if (res.ok) {
-		if (res.body.data) {
-			return scrapeScene(res.body.data, url, site, baseRelease, options);
-		}
+		const dataString = res.item.query.html('#__NEXT_DATA__');
+		const data = dataString && JSON.parse(dataString);
 
-		return null;
+		return scrapeScene(data.props.pageProps, url, channel, options);
 	}
 
 	return res.status;
@@ -415,6 +424,7 @@ async function fetchProfile({ name: actorName }, { site }, include) {
 }
 
 module.exports = {
+	// fetchLatest: fetchLatestGraphql,
 	fetchLatest,
 	fetchUpcoming,
 	fetchScene,

src/scrapers/xempire.js

@@ -1,25 +0,0 @@
-'use strict';
-
-const { fetchLatest, fetchUpcoming, scrapeScene, fetchProfile } = require('./gamma');
-const qu = require('../utils/qu');
-
-async function fetchScene(url, site, baseRelease, options) {
-	const res = await qu.get(url);
-	const release = await scrapeScene(res.item, url, site, baseRelease, null, options);
-
-	const siteDomain = release.query.el('meta[name="twitter:domain"]', 'content') || 'allblackx.com'; // only AllBlackX has no twitter domain, no other useful hints available
-	const siteSlug = siteDomain && siteDomain.split('.')[0].toLowerCase();
-	// const siteUrl = siteDomain && `https://www.${siteDomain}`;
-
-	release.channel = siteSlug;
-	release.director = 'Mason';
-
-	return release;
-}
-
-module.exports = {
-	fetchLatest,
-	fetchProfile,
-	fetchUpcoming,
-	fetchScene,
-};

src/store-releases.js

@@ -38,7 +38,6 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
 		date_precision: release.datePrecision,
 		slug,
 		description: release.description,
-		qualities: release.qualities?.map(Number).filter(Boolean),
 		comment: release.comment,
 		deep: typeof release.deep === 'boolean' ? release.deep : false,
 		deep_url: release.deepUrl,
@@ -49,6 +48,7 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce
 		curatedRelease.shoot_id = release.shootId || null;
 		curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null;
 		curatedRelease.duration = release.duration;
+		curatedRelease.qualities = Array.from(new Set(release.qualities?.map(Number).filter(Boolean)));
 	}
 
 	if (release.productionLocation) {

src/updates.js

@@ -75,8 +75,6 @@ async function filterUniqueReleases(releases) {
 function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
 	const { localUniqueReleases: uniquePageReleases } = filterLocalUniqueReleases(pageReleases, accReleases);
 
-	console.log(uniquePageReleases.length, unextracted.length);
-
 	if (uniquePageReleases.length + unextracted.length === 0) {
 		// page is empty, or only contains scenes from previous page
 		return false;

src/utils/http.js

@@ -28,7 +28,7 @@ const defaultOptions = {
 	encodeJSON: true,
 	parse: false,
 	headers: {
-		'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
+		'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
 	},
 };
 
@@ -163,11 +163,15 @@ function getTimeout(options, url) {
 }
 
 async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
+	if (typeof url !== 'string') {
+		console.trace(`Bad URL: ${JSON.stringify(url)}`);
+	}
+
 	const options = {
 		...defaultOptions,
 		...requestOptions,
 		headers: {
-			...defaultOptions.headers,
+			...(requestOptions.includeDefaultHeaders === false ? {} : defaultOptions.headers),
 			...requestOptions.headers,
 		},
 		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,