DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Storing actor avatars. Using 1 second interval queue for location resolve as per OSM code of conduct.

Browse Source
This commit is contained in:
ThePendulum
2020-05-16 04:36:45 +02:00
parent 21d4dd6bfa
commit 05ee57378a
28 changed files with 899 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/utils/http.js
View File

@@ -36,17 +36,13 @@ function useProxy(url) {
const queue = taskQueue();
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
});
queue.define('http', async ({
async function handler({
url,
method = 'GET',
body,
headers = {},
options = {},
}) => {
}) {
if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${JSON.stringify(body)}`);
} else {
@@ -55,12 +51,12 @@ queue.define('http', async ({
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...(options?.defaultHeaders !== false && defaultHeaders),
...headers,
},
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
...(options?.timeout && { responseTimeout: options?.timeout }),
};
if (useProxy(url)) {
@@ -71,8 +67,8 @@ queue.define('http', async ({
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
if (options.stream && options.destination) {
await pipeline(res, ...(options.transforms || []), options.destination);
if (options?.stream && options?.destination) {
await pipeline(res, ...(options?.transforms || []), options?.destination);
}
const html = Buffer.isBuffer(res.body) ? res.body.toString() : null;
@@ -88,12 +84,22 @@ queue.define('http', async ({
code: res.statusCode,
status: res.statusCode,
};
}, {
}
queue.on('concurrencyReached:http', () => {
logger.silly('Queueing requests');
});
queue.define('20p', handler, {
concurrency: 20,
});
async function get(url, headers, options) {
return queue.push('http', {
queue.define('1s', handler, {
interval: 1,
});
async function get(url, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
method: 'GET',
url,
headers,
@@ -101,8 +107,8 @@ async function get(url, headers, options) {
});
}
async function post(url, body, headers, options) {
return queue.push('http', {
async function post(url, body, headers, options, queueMethod = '20p') {
return queue.push(queueMethod, {
method: 'POST',
url,
body,

2
src/utils/resolve-place.js
View File

@@ -12,7 +12,7 @@ async function resolvePlace(query) {
// https://operations.osmfoundation.org/policies/nominatim/
const res = await http.get(`https://nominatim.openstreetmap.org/search/${encodeURI(query)}?format=json&accept-language=en&addressdetails=1`, {
'User-Agent': 'contact at moonloop.adult@protonmail.com',
});
}, null, '1s');
const [item] = res.body;