Added proxy.

This commit is contained in:
2026-08-17 22:48:02 +02:00
parent 5c7a0397c9
commit 941cb280be
18 changed files with 254 additions and 50 deletions
+2 -1
View File
@@ -6,6 +6,7 @@ const blake2 = require('blake2');
const phash = require('sharp-phash');
// const phashDistance = require('sharp-phash/distance');
const { agentFor } = require('../proxy');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').items;
@@ -22,7 +23,7 @@ async function fetchItem(url, attempt, context) {
try {
// throw new Error('Failed it!');
const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) }));
if (res.statusCode !== 200) {
throw new Error(`Response not OK for ${url} (${res.statusCode})`);