DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Disabled MindGeek session bundling to analyze Too Many Requests errors.

This commit is contained in:
DebaucheryLibrarian 2022-01-16 22:24:47 +01:00
parent 4c503a3177
commit 372db86927
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/scrapers/mindgeek.js
View File

@ -179,8 +179,6 @@ async function getSession(site, parameters) {
const cookieString = await cookieJar.getCookieStringAsync(sessionUrl); const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
const { instance_token: instanceToken } = cookie.parse(cookieString); const { instance_token: instanceToken } = cookie.parse(cookieString);
console.log(site.name, instanceToken);
return { session, instanceToken }; return { session, instanceToken };
} }
@ -240,9 +238,8 @@ async function fetchLatest(site, page = 1, options) {
const { searchParams } = new URL(url); const { searchParams } = new URL(url);
const siteId = searchParams.get('site'); const siteId = searchParams.get('site');
const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); // const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
const { session, instanceToken } = await getSession(site, options.parameters);
console.log('fetch', instanceToken);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 24; const limit = 24;
@ -368,7 +365,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
} }
module.exports = { module.exports = {
beforeNetwork: getSession, // beforeNetwork: getSession,
beforeFetchScenes: getSession, beforeFetchScenes: getSession,
scrapeLatestX, scrapeLatestX,
fetchLatest, fetchLatest,