Added CF resolver to http module. Using priority lookup in tags seed.

assets/components/networks/networks.vue

@@ -106,6 +106,7 @@ function popularEntities() {
 		'pornpros',
 		'private',
 		'realitykings',
+		'teamskeet',
 		'twistys',
 		'vixen',
 		'xempire',

assets/components/releases/release.vue

@@ -204,7 +204,7 @@
 			</div>
 
 			<div
-				v-if="release.qualities"
+				v-if="release.qualities?.length > 0"
 				class="row"
 			>
 				<span class="row-label">Available qualities</span>

config/default.js

@@ -292,6 +292,13 @@ module.exports = {
 			'im0.imgcm.com',
 		],
 	},
+	bypass: {
+		cloudflare: {
+			enable: true,
+			auto: true, // try bypass when CF challenge is detected
+			path: 'http://localhost:8191/v1',
+		},
+	},
 	limits: {
 		default: {
 			interval: 50,

seeds/00_tags.js

@@ -99,7 +99,6 @@ const tags = [
 		name: 'airtight',
 		slug: 'airtight',
 		description: 'Stuffing one cock in your ass, one in your pussy, and one in your mouth, filling up all of your penetrable holes and getting sealed airtight like a figurative balloon. In other words, simultaneously getting [double penetrated](/tag/dp), and giving a [blowjob](/tag/blowjob) or getting [facefucked](/tag/facefucking). Being airtight implies being [gangbanged](/tag/gangbang).', /* eslint-disable-line max-len */
-		priority: 9,
 		group: 'penetration',
 	},
 	{
@@ -109,7 +108,6 @@ const tags = [
 	{
 		name: 'anal creampie',
 		slug: 'anal-creampie',
-		priority: 7,
 		description: 'Ejaculating into the asshole.',
 		group: 'finish',
 	},
@@ -117,7 +115,6 @@ const tags = [
 		name: 'anal',
 		slug: 'anal',
 		description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.',
-		priority: 9,
 		group: 'penetration',
 	},
 	{
@@ -141,7 +138,6 @@ const tags = [
 	{
 		name: 'asian',
 		slug: 'asian',
-		priority: 5,
 		group: 'ethnicity',
 	},
 	{
@@ -152,14 +148,12 @@ const tags = [
 	{
 		name: 'ass to mouth',
 		slug: 'atm',
-		priority: 6,
 		description: 'Sucking off a cock right after it fucked your ass. If it has been in someone else\'s ass, it may be called [ATOGM](/tag/atogm).',
 		group: 'oral',
 	},
 	{
 		name: 'ass to other girl\'s mouth',
 		slug: 'atogm',
-		priority: 6,
 		description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".',
 		group: 'oral',
 	},
@@ -167,7 +161,6 @@ const tags = [
 		name: 'ass eating',
 		slug: 'ass-eating',
 		group: 'oral',
-		priority: 6,
 	},
 	{
 		name: 'ass worship',
@@ -191,7 +184,6 @@ const tags = [
 	{
 		name: 'BDSM',
 		slug: 'bdsm',
-		priority: 8,
 	},
 	{
 		name: 'BBC',
@@ -201,7 +193,6 @@ const tags = [
 	{
 		name: 'behind the scenes',
 		slug: 'bts',
-		priority: 6,
 	},
 	{
 		name: 'big dick',
@@ -221,7 +212,6 @@ const tags = [
 	{
 		name: 'bisexual',
 		slug: 'bisexual',
-		priority: 10,
 	},
 	{
 		name: 'black cock',
@@ -246,14 +236,12 @@ const tags = [
 	{
 		name: 'blowjob',
 		slug: 'blowjob',
-		priority: 5,
 		description: 'Taking a dick in your mouth, sucking, licking and kissing it, often while giving a [handjob](/tag/handjob). You may slide it all the way [down your throat](/tag/deepthroat), or let them [fuck your face](/tag/facefucking).',
 		group: 'oral',
 	},
 	{
 		name: 'blowbang',
 		slug: 'blowbang',
-		priority: 9,
 		description: 'Pleasuring a gang of three or more cocks by sucking and jerking off as many cocks as you can, often getting [facefucked](/tag/facefucking), groped and rubbed out, and followed by a [bukkake](/tag/bukkake). If you are also getting fucked, it is a [gangbang](/tag/gangbang).',
 		group: 'group',
 	},
@@ -273,7 +261,6 @@ const tags = [
 	{
 		name: 'bukkake',
 		slug: 'bukkake',
-		priority: 8,
 		description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
 		group: 'finish',
 	},
@@ -305,7 +292,6 @@ const tags = [
 	{
 		name: 'creampie',
 		slug: 'creampie',
-		priority: 8,
 		description: 'Ejaculalating into her pussy, often shown visibly dripping out afterwards.',
 		group: 'finish',
 	},
@@ -355,64 +341,54 @@ const tags = [
 		name: 'double anal',
 		slug: 'dap',
 		description: 'Two cocks filling up your ass at the same time. If there\'s a third cock in your pussy, it is [double anal triple penetration](/tag/da-tp).',
-		priority: 8,
 		group: 'penetration',
 	},
 	{
 		name: 'double dildo',
 		slug: 'double-dildo',
 		description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).',
-		priority: 4,
 	},
 	{
 		name: 'double dildo anal',
 		slug: 'double-dildo-anal',
 		description: 'Two people ass-fucking eachother with either end of a [double-sided dildo](/tag/double-dildo), "ass to ass".',
-		priority: 4,
 	},
 	{
 		name: 'double dildo DP',
 		slug: 'double-dildo-dp',
 		description: 'Using a [double-sided dildo](/tag/double-dildo) on your ass and pussy [at the same time](/tag/dp).',
-		priority: 4,
 	},
 	{
 		name: 'double dildo blowjob',
 		slug: 'double-dildo-blowjob',
 		description: 'Two people sucking and gagging on either end of a [double-sided dildo](/tag/double-dildo). They may deepthroat the dildo for a [double dildo kiss](/tag/double-dildo-kiss).',
-		priority: 4,
 	},
 	{
 		name: 'double dildo kiss',
 		slug: 'double-dildo-kiss',
 		description: 'Deepthroating a [double-sided dildo](/tag/double-dildo) during a [double dildo blowjob](/tag/double-dildo-blowjob), all the way until you can kiss eachother\'s lips.',
-		priority: 4,
 	},
 	{
 		name: 'triple anal',
 		slug: 'tap',
 		description: 'Getting fucked in the ass by not one, two, but *three* cocks at the same time.',
-		priority: 8,
 		group: 'penetration',
 	},
 	{
 		name: 'triple vaginal',
 		slug: 'tvp',
 		description: 'Getting fucked in the pussy by *three* cocks at the same time.',
-		priority: 8,
 		group: 'penetration',
 	},
 	{
 		name: 'deepthroat',
 		slug: 'deepthroat',
-		priority: 6,
 		description: 'Shoving a cock down your throat during a [blowjob](/tag/blowjob) or [facefuck](/tag/facefucking), giving them a tight sensation while showing off your skills. Without practice, a cock hitting the back of your mouth will likely make you [gag](/tag/gagging).',
 		group: 'oral',
 	},
 	{
 		name: 'double penetration',
 		slug: 'dp',
-		priority: 9,
 		description: 'Getting your [ass](/tag/anal) and pussy fucked at the same time. If you take another cock in your mouth, you are [airtight](/tag/airtight).',
 		group: 'penetration',
 	},
@@ -424,7 +400,6 @@ const tags = [
 		name: 'double vaginal',
 		slug: 'dvp',
 		description: 'Getting fucked with two cocks in your pussy at the same time. If there\'s a third cock in your asshole, it is [double vaginal triple penetration](/tag/dv-tp).',
-		priority: 8,
 		group: 'penetration',
 	},
 	{
@@ -450,7 +425,6 @@ const tags = [
 	{
 		name: 'black',
 		slug: 'black',
-		priority: 5,
 		group: 'ethnicity',
 	},
 	{
@@ -468,7 +442,6 @@ const tags = [
 	{
 		name: 'facefucking',
 		slug: 'facefucking',
-		priority: 7,
 		description: 'A [blowjob](/tag/blowjob) where you give up control, and let them fuck your mouth and [throat](/tag/deepthroat) as if it\'s your pussy.',
 		group: 'oral',
 	},
@@ -485,7 +458,6 @@ const tags = [
 	{
 		name: 'family',
 		slug: 'family',
-		priority: 7,
 	},
 	{
 		name: 'feet',
@@ -519,7 +491,6 @@ const tags = [
 	{
 		name: 'MFF threesome',
 		slug: 'mff',
-		priority: 9,
 		description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
 		group: 'group',
 	},
@@ -535,13 +506,11 @@ const tags = [
 		name: 'gangbang',
 		slug: 'gangbang',
 		description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
-		priority: 9,
 		group: 'group',
 	},
 	{
 		name: 'older men',
 		slug: 'older-men',
-		priority: 7,
 		group: 'age',
 	},
 	{
@@ -562,18 +531,15 @@ const tags = [
 		name: 'trainbang',
 		slug: 'trainbang',
 		description: 'Getting fucked by a group of three or more guys taking turns in a [gangbang](/tag/gangbang), one cock after the other, and never more than one at a time.',
-		priority: 7,
 		group: 'group',
 	},
 	{
 		name: 'gaping',
 		slug: 'gaping',
-		priority: 6,
 	},
 	{
 		name: 'gay',
 		slug: 'gay',
-		priority: 10,
 	},
 	{
 		name: 'gloryhole',
@@ -620,7 +586,6 @@ const tags = [
 	{
 		name: 'interracial',
 		slug: 'interracial',
-		priority: 7,
 		group: 'ethnicity',
 	},
 	{
@@ -634,7 +599,6 @@ const tags = [
 	{
 		name: 'Latina',
 		slug: 'latina',
-		priority: 5,
 		group: 'ethnicity',
 	},
 	{
@@ -644,7 +608,6 @@ const tags = [
 	{
 		name: 'lesbian',
 		slug: 'lesbian',
-		priority: 9,
 	},
 	{
 		name: 'machine dildo',
@@ -671,7 +634,6 @@ const tags = [
 	{
 		name: 'male focus',
 		slug: 'male-focus',
-		priority: 7,
 		description: 'Straight porn highlighting the male talent.',
 	},
 	{
@@ -681,13 +643,11 @@ const tags = [
 	{
 		name: 'MILF',
 		slug: 'milf',
-		priority: 7,
 		group: 'age',
 	},
 	{
 		name: 'MFM threesome',
 		slug: 'mfm',
-		priority: 9,
 		description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
 		group: 'group',
 	},
@@ -731,21 +691,18 @@ const tags = [
 	{
 		name: 'cum in mouth',
 		slug: 'cum-in-mouth',
-		priority: 7,
 		description: 'A guy ejaculating in someone\'s mouth. If they keep their lips wrapped around his cock, it is an [oral creampie](/tag/oral-creampie). They may not be able to resist [swallowing](/tag/swallowing) the cum.',
 		group: 'finish',
 	},
 	{
 		name: 'oral creampie',
 		slug: 'oral-creampie',
-		priority: 7,
 		description: 'A guy pumping his semen in someone\'s *closed* [mouth](/tag/cum-in-mouth), a variation of [cum in mouth](/tag/cum-in-mouth). His cock can be [deepthroated](/tag/deepthroat) for a [throatpie](/tag/throatpie)',
 		group: 'finish',
 	},
 	{
 		name: 'orgy',
 		slug: 'orgy',
-		priority: 9,
 		description: 'A group of (at least four) people having sex with eachother. If only one person is getting fucked, it is probably a [gangbang](/tag/gangbang).',
 		group: 'group',
 	},
@@ -781,12 +738,10 @@ const tags = [
 	{
 		name: 'piss drinking',
 		slug: 'piss-drinking',
-		priority: 6,
 	},
 	{
 		name: 'pissing',
 		slug: 'pissing',
-		priority: 8,
 	},
 	{
 		name: 'POV',
@@ -800,7 +755,6 @@ const tags = [
 	{
 		name: 'pussy to mouth',
 		slug: 'pussy-to-mouth',
-		priority: 5,
 		description: 'Sucking off a cock right fresh out of your pussy.',
 		group: 'oral',
 	},
@@ -829,7 +783,6 @@ const tags = [
 	{
 		name: 'rough',
 		slug: 'rough',
-		priority: 6,
 	},
 	{
 		name: 'saliva',
@@ -939,7 +892,6 @@ const tags = [
 		name: 'swallowing',
 		slug: 'swallowing',
 		group: 'finish',
-		priority: 6,
 	},
 	{
 		name: 'swinging',
@@ -959,13 +911,11 @@ const tags = [
 	{
 		name: 'threesome',
 		slug: 'threesome',
-		priority: 8,
 		group: 'group',
 	},
 	{
 		name: 'throatpie',
 		slug: 'throatpie',
-		priority: 6,
 		description: 'An [oral creampie](/tag/oral-creampie) with his cock shoved all the way down the throat.',
 		group: 'finish',
 	},
@@ -982,43 +932,36 @@ const tags = [
 	{
 		name: 'toys',
 		slug: 'toys',
-		priority: 6,
 	},
 	{
 		name: 'toy anal',
 		slug: 'toy-anal',
 		description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
-		priority: 7,
 	},
 	{
 		name: 'toy DP',
 		slug: 'toy-dp',
 		description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
-		priority: 4,
 	},
 	{
 		name: 'transsexual',
 		slug: 'transsexual',
-		priority: 10,
 	},
 	{
 		name: 'DA triple penetration',
 		slug: 'da-tp',
-		priority: 7,
 		description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your ass](/tag/dap), and one in your pussy. Also see [double vaginal triple penetration](/tag/dv-tp).',
 		group: 'penetration',
 	},
 	{
 		name: 'DV triple penetration',
 		slug: 'dv-tp',
-		priority: 7,
 		description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your pussy](/tag/dvp), and one in [your ass](/tag/anal). Also see [double anal triple penetration](/tag/da-tp).',
 		group: 'penetration',
 	},
 	{
 		name: 'triple penetration',
 		slug: 'triple-penetration',
-		priority: 7,
 		description: 'Three cocks fucking her from behind at the same time. This can be either [double anal TP](/tag/da-tp), or [double vaginal TP](/tag/dv-tp).',
 	},
 	{
@@ -1054,7 +997,6 @@ const tags = [
 	{
 		name: 'Russian',
 		slug: 'russian',
-		priority: 4,
 		group: 'ethnicity',
 	},
 	{
@@ -1064,7 +1006,6 @@ const tags = [
 	{
 		name: 'white',
 		slug: 'white',
-		priority: 7,
 		group: 'ethnicity',
 	},
 	{
@@ -1551,6 +1492,10 @@ const aliases = [
 		for: 'tvp',
 		secondary: true,
 	},
+	{
+		name: 'double vag',
+		for: 'dvp',
+	},
 	{
 		name: 'double vaginal penetration',
 		for: 'dvp',
@@ -1653,6 +1598,10 @@ const aliases = [
 		name: 'french kissing',
 		for: 'kissing',
 	},
+	{
+		name: 'gang bang',
+		for: 'gangbang',
+	},
 	{
 		name: 'gape',
 		for: 'gaping',
@@ -2049,6 +1998,27 @@ const aliases = [
 	},
 ];
 
+const priorities = [ // higher index is higher priority
+	['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
+	['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
+	['family'],
+	['blowjob', 'deepthroat'],
+	['asian', 'black', 'latina', 'white', 'interracial'],
+	['bdsm', 'bts'],
+	['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'],
+	['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
+	['lesbian', 'rough', 'milf', 'male-focus'],
+	['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
+	['anal', 'bukkake'],
+	['dp', 'dap', 'tap', 'dvp', 'tvp', 'airtight'],
+	['gangbang', 'blowbang', 'orgy'],
+	['gay', 'bisexual'],
+].reduce((acc, slugs, index) => {
+	slugs.forEach((slug) => { acc[slug] = index; });
+
+	return acc;
+}, {});
+
 exports.seed = (knex) => Promise.resolve()
 	.then(async () => upsert('tags_groups', groups, 'slug', knex))
 	.then(async () => {
@@ -2059,7 +2029,8 @@ exports.seed = (knex) => Promise.resolve()
 			name: tag.name,
 			slug: tag.slug || slugify(tag.name),
 			description: tag.description,
-			priority: tag.priority || 0,
+			// priority: tag.priority || 0,
+			priority: typeof priorities[tag.slug] === 'undefined' ? -1 : priorities[tag.slug],
 			group_id: tag.group ? groupsMap[tag.group] : null,
 			alias_for: null,
 		}));

src/scrapers/teamskeet.js

@@ -119,7 +119,11 @@ function scrapeProfile(actor, entity) {
 }
 
 async function fetchLatest(channel, page = 1, { parameters }) {
-	const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
+	const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, {
+		bypassCloudflare: true,
+	});
+
+	console.log(res.status);
 
 	if (res.ok) {
 		return scrapeAll(res.body.hits.hits, channel);

src/utils/http.js

@@ -17,7 +17,14 @@ const virtualConsole = require('./virtual-console')(__filename);
 const argv = require('../argv');
 
 const pipeline = util.promisify(stream.pipeline);
-const limiters = {};
+
+const limiters = {
+	bypass: new Bottleneck({
+		minTime: 1000,
+		maxConcurrent: 1,
+		timeout: 60000,
+	}),
+};
 
 Promise.config({
 	cancellation: true,
@@ -84,6 +91,47 @@ function getLimiter(options = {}, url) {
 	return limiters[interval][concurrency];
 }
 
+function extractJson(solution) {
+	if (solution.headers['content-type'].includes('application/json')) {
+		const { document } = new JSDOM(solution.response, { virtualConsole }).window;
+		const dataString = document.querySelector('body > pre')?.textContent;
+
+		if (dataString) {
+			const data = JSON.parse(dataString);
+
+			return data;
+		}
+	}
+
+	return solution.response;
+}
+
+async function bypassCloudflareRequest(url, method, body, options) {
+	// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
+	const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
+		cmd: `request.${method}`,
+		url,
+		maxTimeout: options.timeout,
+		proxy: useProxy(url) ? {
+			url: `${config.proxy.host}:${config.proxy.port}`,
+		} : null,
+	}, {
+		encodeJSON: true,
+	}));
+
+	if (!res.statusCode === 200 || res.body?.status !== 'ok') {
+		throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
+	}
+
+	const resBody = extractJson(res.body.solution);
+
+	return {
+		body: resBody,
+		statusCode: res.body.solution.status,
+		headers: res.body.solution.headers,
+	};
+}
+
 async function request(method = 'get', url, body, requestOptions = {}, limiter) {
 	const http = requestOptions.session || bhttp;
 
@@ -93,12 +141,17 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	};
 
 	const withProxy = useProxy(url);
+	const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable;
 
 	if (withProxy) {
 		options.agent = proxyAgent;
 	}
 
-	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
+	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
+
+	if (withCloudflareBypass) {
+		return bypassCloudflareRequest(url, method, body, options);
+	}
 
 	const res = await (body
 		? http[method](url, body, options)