DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Allowing HTTP rate limits to be set by configuration or argument.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-22 23:50:24 +01:00
parent 6a5063cf32
commit 3d427f7e1d
4 changed files with 54 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/utils/http.js
View File

@@ -9,6 +9,7 @@ const Bottleneck = require('bottleneck');
const { JSDOM } = require('jsdom');
const logger = require('../logger')(__filename);
const argv = require('../argv');
const pipeline = util.promisify(stream.pipeline);
const limiters = {};
@@ -33,12 +34,31 @@ function useProxy(url) {
}
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
}
function getLimiter(limit = {}) {
const interval = limit.interval === undefined ? config.limits.default.interval : limit.interval;
const concurrency = limit.concurrency === undefined ? config.limits.default.concurrency : limit.concurrency;
function getLimiterValue(prop, options, hostname) {
if (argv[prop] !== undefined) {
return argv[prop];
}
if (options[prop] !== undefined) {
return options[prop];
}
if (config.limits[hostname]?.enable !== false && config.limits[hostname]?.[prop] !== undefined) {
return config.limits[hostname][prop];
}
return config.limits.default[prop];
}
function getLimiter(options = {}, url) {
const { hostname } = new URL(url);
const interval = getLimiterValue('interval', options, hostname);
const concurrency = getLimiterValue('concurrency', options, hostname);
if (!limiters[interval]?.[concurrency]) {
limiters[interval] = limiters[interval] || {};
@@ -52,7 +72,7 @@ function getLimiter(limit = {}) {
return limiters[interval][concurrency];
}
async function request(method = 'get', url, body, requestOptions = {}) {
async function request(method = 'get', url, body, requestOptions = {}, limiter) {
const http = requestOptions.session || bhttp;
const options = {
@@ -60,16 +80,16 @@ async function request(method = 'get', url, body, requestOptions = {}) {
...requestOptions,
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
stream: !!requestOptions.destination,
interval: requestOptions.interval || config.limits.default.interval,
concurrency: requestOptions.concurrency || config.limits.default.concurrency,
session: null,
};
if (useProxy(url)) {
const withProxy = useProxy(url);
if (withProxy) {
options.agent = proxyAgent;
}
logger.debug(`GET (${options.interval}ms/${options.concurrency}p) ${url}`);
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
const res = await (body
? http[method](url, body, options)
@@ -107,7 +127,9 @@ async function request(method = 'get', url, body, requestOptions = {}) {
}
async function scheduleRequest(method = 'get', url, body, options) {
return getLimiter(options || {}).schedule(() => request(method, url, body, options));
const limiter = getLimiter(options, url);
return limiter.schedule(() => request(method, url, body, options, limiter));
}
async function get(url, options) {