Fixed media module trying to fetch invalid source URLs. Added Accidental Gangbang to Adult Time.

assets/js/fragments.js

@@ -431,6 +431,22 @@ const releasesFragment = `
                     }
                 }
             }
+			or: [
+				{
+					entity: {
+						slug: {
+							notEqualTo: "analvids"
+						}
+					}
+				}
+				{
+					studio: {
+						slug: {
+							in: ["giorgiograndi"]
+						}
+					}
+				}
+			],
         },
         first: $limit,
 		offset: $offset,

config/default.js

@@ -297,6 +297,11 @@ module.exports = {
 			enable: true,
 			auto: true, // try bypass when CF challenge is detected
 			path: 'http://localhost:8191/v1',
+			sharedHostnames: [ // these can run in the same browser session
+				'store2.psmcdn.net', // Team Skeet API
+				'www.kink.com',
+			],
+			independentHostnames: [], // these must run in their own browser session
 		},
 	},
 	limits: {

seeds/02_sites.js

@@ -363,6 +363,19 @@ const sites = [
 		},
 	},
 	// ADULT TIME
+	{
+		name: 'Accidental Gangbang',
+		slug: 'accidentalgangbang',
+		url: 'https://accidentalgangbang.com',
+		parent: 'adulttime',
+		tags: ['gangbang'],
+		parameters: {
+			referer: 'https://freetour.adulttime.com/en/join',
+			deep: 'https://21sextury.com/en/video',
+			scene: false,
+			includePhotos: false,
+		},
+	},
 	{
 		name: 'ASMR Fantasy',
 		slug: 'asmrfantasy',

src/app.js

@@ -199,6 +199,8 @@ async function init() {
 		logger.error(error);
 	}
 
+	await http.destroyBypassSessions();
+
 	knex.destroy();
 	done = true;
 }

src/media.js

@@ -96,15 +96,39 @@ function itemsByKey(items, key) {
 	return items.reduce((acc, item) => ({ ...acc, [item[key]]: item }), {});
 }
 
+function isValidUrl(url) {
+	try {
+		const urlObject = new URL(url);
+
+		return !!urlObject;
+	} catch (error) {
+		return false;
+	}
+}
+
 function toBaseSource(rawSource) {
 	if (rawSource && (rawSource.src || (rawSource.extract && rawSource.url) || rawSource.stream)) {
 		const baseSource = {};
 
-		if (rawSource.src) baseSource.src = rawSource.src;
+		if (rawSource.src) {
+			if (!isValidUrl(rawSource.src)) {
+				return null;
+			}
+
+			baseSource.src = rawSource.src;
+		}
+
 		if (rawSource.quality) baseSource.quality = rawSource.quality;
 		if (rawSource.type) baseSource.type = rawSource.type;
 
-		if (rawSource.url) baseSource.url = rawSource.url;
+		if (rawSource.url) {
+			if (!isValidUrl(rawSource.url)) {
+				return null;
+			}
+
+			baseSource.url = rawSource.url;
+		}
+
 		if (rawSource.extract) baseSource.extract = rawSource.extract;
 
 		if (rawSource.expectType) baseSource.expectType = rawSource.expectType;
@@ -135,6 +159,10 @@ function toBaseSource(rawSource) {
 	}
 
 	if (typeof rawSource === 'string') {
+		if (!isValidUrl(rawSource)) {
+			return null;
+		}
+
 		return {
 			src: rawSource,
 		};

src/scrapers/gamma.js

@@ -150,7 +150,7 @@ async function getFullPhotos(entryId, site) {
 		},
 	});
 
-	if (res.ok) {
+	if (res.ok && typeof res.body === 'object') { // gives 200 OK even when redirected to signup page
 		return Object.values(res.body);
 	}
 
@@ -442,7 +442,7 @@ async function scrapeReleaseApi(data, site, options) {
 		];
 	}
 
-	if (data.photoset_id && options.includePhotos) {
+	if (data.photoset_id && options.includePhotos && options.parameters?.includePhotos !== false) {
 		release.photos = await getPhotosApi(data.photoset_id, site, options.parameters);
 	}
 

src/scrapers/kink.js

@@ -132,13 +132,9 @@ async function scrapeProfile({ query }, actorUrl, include) {
 
 async function fetchLatest(site, page = 1) {
 	// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
-	const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
+	// const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
-		Host: 'www.kink.com',
+	const res = await qu.getAll(`https://www.kink.com/search?type=shoots&channelIds=${site.slug}&sort=published&page=${page}`, '.shoot-list .shoot', {
-		'User-Agent': 'HTTPie/2.6.0',
+		ct: 2,
-		'Accept-Encoding': 'gzip, deflate, br',
-		Accept: '*/*',
-		Connection: 'keep-alive',
-
 	}, {
 		includeDefaultHeaders: false,
 		followRedirects: false,

src/scrapers/teamskeet.js

@@ -119,11 +119,7 @@ function scrapeProfile(actor, entity) {
 }
 
 async function fetchLatest(channel, page = 1, { parameters }) {
-	const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, {
+	const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
-		bypassCloudflare: true,
-	});
-
-	console.log(res.status);
 
 	if (res.ok) {
 		return scrapeAll(res.body.hits.hits, channel);

src/utils/http.js

@@ -26,6 +26,8 @@ const limiters = {
 	}),
 };
 
+const bypassSessions = new Map();
+
 Promise.config({
 	cancellation: true,
 });
@@ -56,6 +58,32 @@ function useProxy(url) {
 	return config.proxy.hostnames.includes(hostname);
 }
 
+function useCloudflareBypass(url, options) {
+	if (!config.bypass.cloudflare.enable) {
+		return null;
+	}
+
+	const { hostname } = new URL(url);
+
+	if (options.bypassCloudflare === 'shared') {
+		return 'shared';
+	}
+
+	if (options.bypassCloudflare === 'independent') {
+		return hostname;
+	}
+
+	if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
+		return 'shared';
+	}
+
+	if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
+		return hostname;
+	}
+
+	return null;
+}
+
 function getLimiterValue(prop, options, hostname) {
 	if (argv[prop] !== undefined) {
 		return argv[prop];
@@ -106,11 +134,72 @@ function extractJson(solution) {
 	return solution.response;
 }
 
-async function bypassCloudflareRequest(url, method, body, options) {
+async function getBypassSession(url, hostname) {
+	if (bypassSessions.has(hostname)) {
+		return bypassSessions.get(hostname);
+	}
+
+	const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.create',
+		proxy: useProxy(url) ? {
+			url: `${config.proxy.host}:${config.proxy.port}`,
+		} : null,
+	}, {
+		encodeJSON: true,
+	});
+
+	if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
+		throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
+	}
+
+	bypassSessions.set(hostname, sessionRes.body.session);
+
+	return sessionRes.body.session;
+}
+
+async function destroyBypassSession(sessionId) {
+	const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.destroy',
+		session: sessionId,
+	}, {
+		encodeJSON: true,
+	}));
+
+	if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
+		bypassSessions.delete(sessionId);
+
+		logger.verbose(`Destroyed bypass session ${sessionId}`);
+
+		return true;
+	}
+
+	logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
+
+	return false;
+}
+
+async function destroyBypassSessions() {
+	const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.list',
+	}, {
+		encodeJSON: true,
+	}));
+
+	if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
+		logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
+	}
+
+	await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
+}
+
+async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
+	const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
+
 	// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
 	const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
 		cmd: `request.${method}`,
 		url,
+		session: sessionId,
 		maxTimeout: options.timeout,
 		proxy: useProxy(url) ? {
 			url: `${config.proxy.host}:${config.proxy.port}`,
@@ -120,6 +209,12 @@ async function bypassCloudflareRequest(url, method, body, options) {
 	}));
 
 	if (!res.statusCode === 200 || res.body?.status !== 'ok') {
+		if (/session closed/i.test(res.body?.message) && attempts < 3) {
+			await destroyBypassSession(sessionId);
+
+			return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
+		}
+
 		throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
 	}
 
@@ -141,7 +236,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	};
 
 	const withProxy = useProxy(url);
-	const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable;
+	const withCloudflareBypass = useCloudflareBypass(url, options);
 
 	if (withProxy) {
 		options.agent = proxyAgent;
@@ -150,7 +245,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
 
 	if (withCloudflareBypass) {
-		return bypassCloudflareRequest(url, method, body, options);
+		return bypassCloudflareRequest(url, method, body, withCloudflareBypass, options);
 	}
 
 	const res = await (body
@@ -292,4 +387,5 @@ module.exports = {
 	cookieJar: getCookieJar,
 	getSession,
 	getCookieJar,
+	destroyBypassSessions,
 };