Queueing and batching media HTTP requests for improved reliability.

src/utils/chunk.js

@@ -0,0 +1,8 @@
+'use strict';
+
+function chunk(array, chunkSize) {
+    return Array.from({ length: Math.ceil(array.length / chunkSize) })
+        .map((value, index) => array.slice(index * chunkSize, (index * chunkSize) + chunkSize));
+}
+
+module.exports = chunk;

src/utils/http.js

@@ -0,0 +1,76 @@
+'use strict';
+
+const bhttp = require('bhttp');
+const taskQueue = require('promise-task-queue');
+
+const logger = require('../logger')(__filename);
+
+const queue = taskQueue();
+
+queue.on('concurrencyReached:httpGet', () => {
+    logger.silly('Queueing GET requests');
+});
+
+queue.on('concurrencyReached:httpPost', () => {
+    logger.silly('Queueing POST requests');
+});
+
+queue.define('httpGet', async ({
+    url,
+    timeout = 30000,
+    options = {},
+}) => {
+    logger.silly(`GET ${url}`);
+
+    const res = await bhttp.get(url, {
+        responseTimeout: timeout,
+        ...options,
+    });
+
+    res.code = res.statusCode;
+
+    return res;
+}, {
+    concurrency: 20,
+});
+
+queue.define('httpPost', async ({
+    url,
+    body,
+    timeout = 30000,
+    options = {},
+}) => {
+    logger.silly(`POST ${url} with ${body}`);
+
+    const res = await bhttp.post(url, body, {
+        responseTimeout: timeout,
+        ...options,
+    });
+
+    res.code = res.statusCode;
+
+    return res;
+}, {
+    concurrency: 20,
+});
+
+async function get(url, options) {
+    return queue.push('httpGet', {
+        method: 'get',
+        url,
+        options,
+    });
+}
+
+async function post(url, body, options) {
+    return queue.push('httpPost', {
+        url,
+        body,
+        options,
+    });
+}
+
+module.exports = {
+    get,
+    post,
+};

src/utils/q.js

@@ -2,7 +2,7 @@
 
 const { JSDOM } = require('jsdom');
 const moment = require('moment');
-const bhttp = require('bhttp');
+const http = require('./http');
 
 function trim(str) {
     return str.trim().replace(/\s+/g, ' ');
@@ -228,7 +228,7 @@ function extractAll(html, selector) {
 }
 
 async function get(url, selector, headers, all = false) {
-    const res = await bhttp.get(url, {
+    const res = await http.get(url, {
         headers,
     });