forked from DebaucheryLibrarian/traxxx
Merge branch 'master' into experimental
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const util = require('util');
|
||||
const stream = require('stream');
|
||||
@@ -15,7 +16,12 @@ const argv = require('../argv');
|
||||
const pipeline = util.promisify(stream.pipeline);
|
||||
const limiters = {};
|
||||
|
||||
Promise.config({
|
||||
cancellation: true,
|
||||
});
|
||||
|
||||
const defaultOptions = {
|
||||
timeout: 60000,
|
||||
encodeJSON: true,
|
||||
headers: {
|
||||
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
|
||||
@@ -67,6 +73,7 @@ function getLimiter(options = {}, url) {
|
||||
limiters[interval][concurrency] = new Bottleneck({
|
||||
minTime: interval,
|
||||
maxConcurrent: concurrency,
|
||||
timeout: (options.timeout || defaultOptions.timeout) + 10000, // timeout 10 seconds after bhttp should
|
||||
});
|
||||
}
|
||||
|
||||
@@ -83,7 +90,7 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
|
||||
...defaultOptions.headers,
|
||||
...requestOptions.headers,
|
||||
},
|
||||
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
|
||||
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
|
||||
stream: !!requestOptions.destination,
|
||||
session: null,
|
||||
};
|
||||
@@ -131,10 +138,25 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
|
||||
};
|
||||
}
|
||||
|
||||
function getTimeout(options, url) {
|
||||
return new Promise((resolve, reject, onCancel) => {
|
||||
const timeoutId = setTimeout(() => {
|
||||
reject(new Error(`URL ${url} timed out`));
|
||||
}, (options?.timeout || defaultOptions.timeout) + 10000);
|
||||
|
||||
onCancel(() => clearTimeout(timeoutId));
|
||||
});
|
||||
}
|
||||
|
||||
async function scheduleRequest(method = 'get', url, body, options) {
|
||||
const limiter = getLimiter(options, url);
|
||||
const timeout = getTimeout(options, url);
|
||||
|
||||
return limiter.schedule(() => request(method, url, body, options, limiter));
|
||||
const result = await limiter.schedule(() => Promise.race([request(method, url, body, options, limiter), timeout]));
|
||||
|
||||
timeout.cancel();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async function get(url, options) {
|
||||
|
||||
@@ -45,4 +45,5 @@ module.exports = postgraphile(
|
||||
],
|
||||
pgSettings,
|
||||
},
|
||||
pgSettings,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user