diff --git a/src/scrapers/aylo.js b/src/scrapers/aylo.js index 53d39108..57be7cb9 100755 --- a/src/scrapers/aylo.js +++ b/src/scrapers/aylo.js @@ -245,7 +245,6 @@ async function getSession(site, parameters, url) { headers: { 'Accept-Language': 'en-US,en;', // somehow seems essential for some MG sites Connection: 'keep-alive', - 'User-Agent': 'HTTPie/3.2.1', }, interval: parameters?.interval, concurrency: parameters?.concurrency, diff --git a/src/utils/http.js b/src/utils/http.js index a2ba75ba..c9a10df7 100755 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -438,8 +438,8 @@ async function scheduleRequest(method = 'get', url, body, requestOptions = {}) { ...defaultOptions, ...requestOptions, headers: { - ...(requestOptions.includeDefaultHeaders === false ? {} : defaultOptions.headers), - ...requestOptions.headers, + ...Object.fromEntries(requestOptions.includeDefaultHeaders === false ? {} : Object.entries(defaultOptions.headers).map(([key, value]) => [key.toLowerCase(), value])), + ...Object.fromEntries(Object.entries(requestOptions.headers || {}).map(([key, value]) => [key.toLowerCase(), value])), }, responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout, stream: !!requestOptions.destination,