From 372db86927d4417eb556be6f15e04ac1b6c27a9b Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 16 Jan 2022 22:24:47 +0100 Subject: [PATCH] Disabled MindGeek session bundling to analyze Too Many Requests errors. --- src/scrapers/mindgeek.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index 5d7e38a6..f4db2a21 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -179,8 +179,6 @@ async function getSession(site, parameters) { const cookieString = await cookieJar.getCookieStringAsync(sessionUrl); const { instance_token: instanceToken } = cookie.parse(cookieString); - console.log(site.name, instanceToken); - return { session, instanceToken }; } @@ -240,9 +238,8 @@ async function fetchLatest(site, page = 1, options) { const { searchParams } = new URL(url); const siteId = searchParams.get('site'); - const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); - - console.log('fetch', instanceToken); + // const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); + const { session, instanceToken } = await getSession(site, options.parameters); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const limit = 24; @@ -368,7 +365,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para } module.exports = { - beforeNetwork: getSession, + // beforeNetwork: getSession, beforeFetchScenes: getSession, scrapeLatestX, fetchLatest,