Merge branch 'master' into experimental

package-lock.json

@@ -1,11 +1,11 @@
 {
     "name": "traxxx",
-    "version": "1.184.1",
+    "version": "1.184.2",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
-            "version": "1.184.1",
+            "version": "1.184.2",
             "license": "ISC",
             "dependencies": {
                 "@casl/ability": "^5.2.2",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.184.1",
+    "version": "1.184.2",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/utils/http.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const config = require('config');
+const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const util = require('util');
 const stream = require('stream');
@@ -15,7 +16,12 @@ const argv = require('../argv');
 const pipeline = util.promisify(stream.pipeline);
 const limiters = {};
 
+Promise.config({
+	cancellation: true,
+});
+
 const defaultOptions = {
+	timeout: 60000,
 	encodeJSON: true,
 	headers: {
 		'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
@@ -67,6 +73,7 @@ function getLimiter(options = {}, url) {
 		limiters[interval][concurrency] = new Bottleneck({
 			minTime: interval,
 			maxConcurrent: concurrency,
+			timeout: (options.timeout || defaultOptions.timeout) + 10000, // timeout 10 seconds after bhttp should
 		});
 	}
 
@@ -83,7 +90,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 			...defaultOptions.headers,
 			...requestOptions.headers,
 		},
-		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
+		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
 		stream: !!requestOptions.destination,
 		session: null,
 	};
@@ -131,10 +138,25 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
 	};
 }
 
+function getTimeout(options, url) {
+	return new Promise((resolve, reject, onCancel) => {
+		const timeoutId = setTimeout(() => {
+			reject(new Error(`URL ${url} timed out`));
+		}, (options?.timeout || defaultOptions.timeout) + 10000);
+
+		onCancel(() => clearTimeout(timeoutId));
+	});
+}
+
 async function scheduleRequest(method = 'get', url, body, options) {
 	const limiter = getLimiter(options, url);
+	const timeout = getTimeout(options, url);
 
-	return limiter.schedule(() => request(method, url, body, options, limiter));
+	const result = await limiter.schedule(() => Promise.race([request(method, url, body, options, limiter), timeout]));
+
+	timeout.cancel();
+
+	return result;
 }
 
 async function get(url, options) {

src/web/postgraphile.js

@@ -45,4 +45,5 @@ module.exports = postgraphile(
 		],
 		pgSettings,
 	},
+	pgSettings,
 );