Improved update runner. Improved HTTP module API, added default user agent. Added PornCZ and Czechav logos.

src/utils/http.js

@@ -7,6 +7,14 @@ const taskQueue = require('promise-task-queue');
 
 const logger = require('../logger')(__filename);
 
+const defaultHeaders = {
+    'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
+};
+
+const defaultOptions = {
+    responseTimeout: 30000,
+};
+
 const proxyAgent = tunnel.httpsOverHttp({
     proxy: {
         host: config.proxy.host,
@@ -25,19 +33,15 @@ function useProxy(url) {
 
 const queue = taskQueue();
 
-queue.on('concurrencyReached:httpGet', () => {
-    logger.silly('Queueing GET requests');
-});
-
-queue.on('concurrencyReached:httpPost', () => {
-    logger.silly('Queueing POST requests');
+queue.on('concurrencyReached:http', () => {
+    logger.silly('Queueing requests');
 });
 
 queue.define('http', async ({
     url,
     method = 'GET',
     body,
-    timeout = 30000,
+    headers = {},
     options = {},
 }) => {
     if (body) {
@@ -47,8 +51,13 @@ queue.define('http', async ({
     }
 
     const reqOptions = {
-        responseTimeout: timeout,
+        headers: {
+            ...headers,
+            ...defaultHeaders,
+        },
         ...options,
+        ...defaultOptions,
+        ...(options.timeout && { responseTimeout: options.timeout }),
     };
 
     if (useProxy(url)) {
@@ -59,26 +68,33 @@ queue.define('http', async ({
         ? await bhttp[method.toLowerCase()](url, body, reqOptions)
         : await bhttp[method.toLowerCase()](url, reqOptions);
 
+    const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
+    const json = Buffer.isBuffer(res.body) ? null : res.body;
+
     return {
         ...res,
+        html,
+        json,
         code: res.statusCode,
     };
 }, {
     concurrency: 20,
 });
 
-async function get(url, options) {
+async function get(url, headers, options) {
     return queue.push('http', {
         method: 'get',
         url,
+        headers,
         options,
     });
 }
 
-async function post(url, body, options) {
+async function post(url, body, headers, options) {
     return queue.push('http', {
         url,
         body,
+        headers,
         options,
     });
 }