Added proxy.
This commit is contained in:
+2
-1
@@ -6,6 +6,7 @@ const blake2 = require('blake2');
|
||||
const phash = require('sharp-phash');
|
||||
// const phashDistance = require('sharp-phash/distance');
|
||||
|
||||
const { agentFor } = require('../proxy');
|
||||
const logger = require('../logger')(__filename);
|
||||
const limiter = require('../limiter').items;
|
||||
|
||||
@@ -22,7 +23,7 @@ async function fetchItem(url, attempt, context) {
|
||||
|
||||
try {
|
||||
// throw new Error('Failed it!');
|
||||
const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers }));
|
||||
const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers, agent: agentFor(url) }));
|
||||
|
||||
if (res.statusCode !== 200) {
|
||||
throw new Error(`Response not OK for ${url} (${res.statusCode})`);
|
||||
|
||||
Reference in New Issue
Block a user