traxxx/src/utils/http.js

'use strict';

const config = require('config');
const bhttp = require('bhttp');
const util = require('util');
const stream = require('stream');
const tunnel = require('tunnel');
const Bottleneck = require('bottleneck');
const { JSDOM } = require('jsdom');

const logger = require('../logger')(__filename);

const pipeline = util.promisify(stream.pipeline);
const limiters = {};

const defaultOptions = {
	encodeJSON: true,
	headers: {
		'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
	},
};

const proxyAgent = tunnel.httpsOverHttp({
	proxy: {
		host: config.proxy.host,
		port: config.proxy.port,
	},
});

function useProxy(url) {
	if (!config.proxy.enable) {
		return false;
	}

	const { hostname } = new URL(url);
	return config.proxy.hostnames.includes(hostname);
}

function getLimiter(limit = {}) {
	const interval = limit.interval === undefined ? config.limits.default.interval : limit.interval;
	const concurrency = limit.concurrency === undefined ? config.limits.default.concurrency : limit.concurrency;

	if (!limiters[interval]?.[concurrency]) {
		limiters[interval] = limiters[interval] || {};

		limiters[interval][concurrency] = new Bottleneck({
			minTime: interval,
			maxConcurrent: concurrency,
		});
	}

	return limiters[interval][concurrency];
}

async function request(method = 'get', url, body, requestOptions = {}) {
	const http = requestOptions.session || bhttp;

	const options = {
		...defaultOptions,
		...requestOptions,
		responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || 60000,
		stream: !!requestOptions.destination,
		interval: requestOptions.interval || config.limits.default.interval,
		concurrency: requestOptions.concurrency || config.limits.default.concurrency,
		session: null,
	};

	if (useProxy(url)) {
		options.agent = proxyAgent;
	}

	logger.debug(`GET (${options.interval}ms/${options.concurrency}p) ${url}`);

	const res = await (body
		? http[method](url, body, options)
		: http[method](url, options));

	const resIsOk = res.statusCode >= 200 && res.statusCode <= 299;

	if (options.destination) {
		// res.on('progress', (bytes, totalBytes) => logger.silly(`Downloaded ${Math.round((bytes / totalBytes) * 100)}% of ${url}`));

		await pipeline(res, ...(options.transforms || []), options.destination);
	}

	if (Buffer.isBuffer(res.body)) {
		const html = res.body.toString();
		const window = new JSDOM(html).window;

		return {
			...res,
			body: html,
			html,
			status: res.statusCode,
			document: window.document,
			window,
			ok: resIsOk,
		};
	}

	return {
		...res,
		body: res.body,
		status: res.statusCode,
		ok: res.statusCode >= 200 && res.statusCode <= 299,
	};
}

async function scheduleRequest(method = 'get', url, body, options) {
	return getLimiter(options || {}).schedule(() => request(method, url, body, options));
}

async function get(url, options) {
	return scheduleRequest('get', url, null, options);
}

async function post(url, body, options) {
	return scheduleRequest('post', url, body, options);
}

async function put(url, body, options) {
	return scheduleRequest('put', url, body, options);
}

async function patch(url, body, options) {
	return scheduleRequest('patch', url, body, options);
}

async function del(url, options) {
	return scheduleRequest('delete', url, null, options);
}

async function head(url, options) {
	return scheduleRequest('head', url, null, options);
}

function getSession(options) {
	return bhttp.session(options);
}

module.exports = {
	get,
	head,
	post,
	delete: del,
	put,
	patch,
	session: getSession,
};