'use strict'; const util = require('util'); const stream = require('stream'); const config = require('config'); const tunnel = require('tunnel'); const bhttp = require('bhttp'); const taskQueue = require('promise-task-queue'); const pipeline = util.promisify(stream.pipeline); const logger = require('../logger')(__filename); const defaultHeaders = { 'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1', }; const defaultOptions = { responseTimeout: 30000, }; const proxyAgent = tunnel.httpsOverHttp({ proxy: { host: config.proxy.host, port: config.proxy.port, }, }); function useProxy(url) { if (!config.proxy.enable) { return false; } const { hostname } = new URL(url); return config.proxy.hostnames.includes(hostname); } const queue = taskQueue(); queue.on('concurrencyReached:http', () => { logger.silly('Queueing requests'); }); queue.define('http', async ({ url, method = 'GET', body, headers = {}, options = {}, }) => { if (body) { logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`); } else { logger.silly(`${method.toUpperCase()} ${url}`); } const reqOptions = { headers: { ...(options.defaultHeaders !== false && defaultHeaders), ...headers, }, ...defaultOptions, ...options, ...(options.timeout && { responseTimeout: options.timeout }), }; if (useProxy(url)) { reqOptions.agent = proxyAgent; } const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase()) ? await bhttp[method.toLowerCase()](url, body, reqOptions) : await bhttp[method.toLowerCase()](url, reqOptions); if (options.stream && options.destination) { await pipeline(res, ...(options.transforms || []), options.destination); } const html = Buffer.isBuffer(res.body) ? res.body.toString() : null; const json = Buffer.isBuffer(res.body) ? null : res.body; return { ...res, originalRes: res, html, json, pipe: res.pipe, ok: res.statusCode >= 200 && res.statusCode <= 299, code: res.statusCode, status: res.statusCode, }; }, { concurrency: 20, }); async function get(url, headers, options) { return queue.push('http', { method: 'GET', url, headers, options, }); } async function post(url, body, headers, options) { return queue.push('http', { method: 'POST', url, body, headers, options, }); } module.exports = { get, post, };