Added proxy support to HTTP module. Added Vixen hostnames to default proxy config.

config/default.js

@@ -12,9 +12,44 @@ module.exports = {
     // include: [],
     // exclude: [],
     exclude: [
+        ['21sextreme', [
+            // no longer updated
+            'mightymistress',
+            'dominatedgirls',
+            'homepornreality',
+            'peeandblow',
+            'cummingmatures',
+            'mandyiskinky',
+            'speculumplays',
+            'creampiereality',
+        ]],
+        ['blowpass', ['sunlustxxx']],
+        ['ddfnetwork', [
+            'fuckinhd',
+            'bustylover',
+        ]],
         ['famedigital', [
+            'daringsex',
             'lowartfilms',
         ]],
+        ['pornpros', [
+            'milfhumiliation',
+            'humiliated',
+            'flexiblepositions',
+            'publicviolations',
+            'amateurviolations',
+            'squirtdisgrace',
+            'cumdisgrace',
+            'webcamhackers',
+            'collegeteens',
+        ]],
+        ['score', [
+            'bigboobbundle',
+            'milfbundle',
+            'pornmegaload',
+            'scorelandtv',
+            'scoretv',
+        ]],
     ],
     profiles: [
         [
@@ -104,6 +139,19 @@ module.exports = {
         'freeones',
         'freeonesLegacy',
     ],
+    proxy: {
+        enable: false,
+        host: '',
+        port: 8888,
+        hostnames: [
+            'www.vixen.com',
+            'www.blacked.com',
+            'www.blackedraw.com',
+            'www.tushy.com',
+            'www.tushyraw.com',
+            'www.deeper.com',
+        ],
+    },
     fetchAfter: [1, 'week'],
     nullDateLimit: 3,
     media: {

src/utils/http.js

@@ -1,10 +1,28 @@
 'use strict';
 
+const config = require('config');
+const tunnel = require('tunnel');
 const bhttp = require('bhttp');
 const taskQueue = require('promise-task-queue');
 
 const logger = require('../logger')(__filename);
 
+const proxyAgent = tunnel.httpsOverHttp({
+    proxy: {
+        host: config.proxy.host,
+        port: config.proxy.port,
+    },
+});
+
+function useProxy(url) {
+    if (!config.proxy.enable) {
+        return false;
+    }
+
+    const { hostname } = new URL(url);
+    return config.proxy.hostnames.includes(hostname);
+}
+
 const queue = taskQueue();
 
 queue.on('concurrencyReached:httpGet', () => {
@@ -15,47 +33,42 @@ queue.on('concurrencyReached:httpPost', () => {
     logger.silly('Queueing POST requests');
 });
 
-queue.define('httpGet', async ({
-    url,
-    timeout = 30000,
-    options = {},
-}) => {
-    logger.silly(`GET ${url}`);
-
-    const res = await bhttp.get(url, {
-        responseTimeout: timeout,
-        ...options,
-    });
-
-    res.code = res.statusCode;
-
-    return res;
-}, {
-    concurrency: 20,
-});
-
-queue.define('httpPost', async ({
+queue.define('http', async ({
     url,
+    method = 'GET',
     body,
     timeout = 30000,
     options = {},
 }) => {
-    logger.silly(`POST ${url} with ${body}`);
+    if (body) {
+        logger.silly(`${method.toUpperCase()} ${url} with ${body}`);
+    } else {
+        logger.silly(`${method.toUpperCase()} ${url}`);
+    }
 
-    const res = await bhttp.post(url, body, {
+    const reqOptions = {
         responseTimeout: timeout,
         ...options,
-    });
+    };
 
-    res.code = res.statusCode;
+    if (useProxy(url)) {
+        reqOptions.agent = proxyAgent;
+    }
 
-    return res;
+    const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
+        ? await bhttp[method.toLowerCase()](url, body, reqOptions)
+        : await bhttp[method.toLowerCase()](url, reqOptions);
+
+    return {
+        ...res,
+        code: res.statusCode,
+    };
 }, {
     concurrency: 20,
 });
 
 async function get(url, options) {
-    return queue.push('httpGet', {
+    return queue.push('http', {
         method: 'get',
         url,
         options,
@@ -63,7 +76,7 @@ async function get(url, options) {
 }
 
 async function post(url, body, options) {
-    return queue.push('httpPost', {
+    return queue.push('http', {
         url,
         body,
         options,