DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

Browse Source
This commit is contained in:
ThePendulum
2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
src/utils/http.js
View File

@@ -11,107 +11,107 @@ const pipeline = util.promisify(stream.pipeline);
const logger = require('../logger')(__filename);
const defaultHeaders = {
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
};
const defaultOptions = {
responseTimeout: 30000,
responseTimeout: 30000,
};
const proxyAgent = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: config.proxy.port,
},
proxy: {
host: config.proxy.host,
port: config.proxy.port,
},
});
function useProxy(url) {
if (!config.proxy.enable) {
return false;
}
if (!config.proxy.enable) {
return false;
}
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
}
const queue = taskQueue();
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
logger.silly('Queueing requests');
});
queue.define('http', async ({
url,
method = 'GET',
body,
headers = {},
options = {},
url,
method = 'GET',
body,
headers = {},
options = {},
}) => {
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
}
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
}
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
};
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
};
if (useProxy(url)) {
reqOptions.agent = proxyAgent;
}
if (useProxy(url)) {
reqOptions.agent = proxyAgent;
}
const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
}
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
}
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
const json = Buffer.isBuffer(res.body) ? null : res.body;
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
const json = Buffer.isBuffer(res.body) ? null : res.body;
return {
...res,
originalRes: res,
html,
json,
pipe: res.pipe,
ok: res.statusCode >= 200 && res.statusCode <= 299,
code: res.statusCode,
status: res.statusCode,
};
return {
...res,
originalRes: res,
html,
json,
pipe: res.pipe,
ok: res.statusCode >= 200 && res.statusCode <= 299,
code: res.statusCode,
status: res.statusCode,
};
}, {
concurrency: 20,
concurrency: 20,
});
async function get(url, headers, options) {
return queue.push('http', {
method: 'GET',
url,
headers,
options,
});
return queue.push('http', {
method: 'GET',
url,
headers,
options,
});
}
async function post(url, body, headers, options) {
return queue.push('http', {
method: 'POST',
url,
body,
headers,
options,
});
return queue.push('http', {
method: 'POST',
url,
body,
headers,
options,
});
}
module.exports = {
get,
post,
get,
post,
};