DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using UTC to query date ranges. Removed stray console log from MindGeek scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-08-26 02:01:38 +02:00
parent 52f66e7982
commit 8611d738b0
6 changed files with 19 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/scrapers/mindgeek.js
View File

@@ -146,12 +146,16 @@ async function getSession(url) {
const cookieJar = new CookieJar();
const session = bhttp.session({ cookieJar });
await session.get(url);
const res = await session.get(url);
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
if (res.statusCode === 200) {
const cookieString = await cookieJar.getCookieStringAsync(url);
const { instance_token: instanceToken } = cookieToData(cookieString);
return { session, instanceToken };
return { session, instanceToken };
}
throw new Error(`Failed to acquire MindGeek session: ${res.statusCode}`);
}
function scrapeProfile(data, html, releases = [], networkName) {
@@ -242,11 +246,14 @@ async function fetchUpcoming(site) {
}
async function fetchScene(url, site, baseScene) {
if (baseScene?.entryId) {
// overview and deep data is the same, don't hit server unnecessarily
return baseScene;
}
const entryId = url.match(/\d+/)[0];
const { session, instanceToken } = await getSession(url);
console.log(baseScene);
const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
headers: {
Instance: instanceToken,