Fixed media module trying to fetch invalid source URLs. Added Accidental Gangbang to Adult Time.

src/utils/http.js

@@ -26,6 +26,8 @@ const limiters = {
 	}),
 };
 
+const bypassSessions = new Map();
+
 Promise.config({
 	cancellation: true,
 });
@@ -56,6 +58,32 @@ function useProxy(url) {
 	return config.proxy.hostnames.includes(hostname);
 }
 
+function useCloudflareBypass(url, options) {
+	if (!config.bypass.cloudflare.enable) {
+		return null;
+	}
+
+	const { hostname } = new URL(url);
+
+	if (options.bypassCloudflare === 'shared') {
+		return 'shared';
+	}
+
+	if (options.bypassCloudflare === 'independent') {
+		return hostname;
+	}
+
+	if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
+		return 'shared';
+	}
+
+	if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
+		return hostname;
+	}
+
+	return null;
+}
+
 function getLimiterValue(prop, options, hostname) {
 	if (argv[prop] !== undefined) {
 		return argv[prop];
@@ -106,11 +134,72 @@ function extractJson(solution) {
 	return solution.response;
 }
 
-async function bypassCloudflareRequest(url, method, body, options) {
+async function getBypassSession(url, hostname) {
+	if (bypassSessions.has(hostname)) {
+		return bypassSessions.get(hostname);
+	}
+
+	const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.create',
+		proxy: useProxy(url) ? {
+			url: `${config.proxy.host}:${config.proxy.port}`,
+		} : null,
+	}, {
+		encodeJSON: true,
+	});
+
+	if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
+		throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
+	}
+
+	bypassSessions.set(hostname, sessionRes.body.session);
+
+	return sessionRes.body.session;
+}
+
+async function destroyBypassSession(sessionId) {
+	const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.destroy',
+		session: sessionId,
+	}, {
+		encodeJSON: true,
+	}));
+
+	if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
+		bypassSessions.delete(sessionId);
+
+		logger.verbose(`Destroyed bypass session ${sessionId}`);
+
+		return true;
+	}
+
+	logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
+
+	return false;
+}
+
+async function destroyBypassSessions() {
+	const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
+		cmd: 'sessions.list',
+	}, {
+		encodeJSON: true,
+	}));
+
+	if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
+		logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
+	}
+
+	await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
+}
+
+async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
+	const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
+
 	// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
 	const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
 		cmd: `request.${method}`,
 		url,
+		session: sessionId,
 		maxTimeout: options.timeout,
 		proxy: useProxy(url) ? {
 			url: `${config.proxy.host}:${config.proxy.port}`,
@@ -120,6 +209,12 @@ async function bypassCloudflareRequest(url, method, body, options) {
 	}));
 
 	if (!res.statusCode === 200 || res.body?.status !== 'ok') {
+		if (/session closed/i.test(res.body?.message) && attempts < 3) {
+			await destroyBypassSession(sessionId);
+
+			return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
+		}
+
 		throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
 	}
 
@@ -141,7 +236,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	};
 
 	const withProxy = useProxy(url);
-	const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable;
+	const withCloudflareBypass = useCloudflareBypass(url, options);
 
 	if (withProxy) {
 		options.agent = proxyAgent;
@@ -150,7 +245,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
 
 	if (withCloudflareBypass) {
-		return bypassCloudflareRequest(url, method, body, options);
+		return bypassCloudflareRequest(url, method, body, withCloudflareBypass, options);
 	}
 
 	const res = await (body
@@ -292,4 +387,5 @@ module.exports = {
 	cookieJar: getCookieJar,
 	getSession,
 	getCookieJar,
+	destroyBypassSessions,
 };