From 7f8704ee2c3705d04f8f77602372040a7224a713 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Thu, 10 Sep 2020 17:41:29 +0200 Subject: [PATCH] Using http instead of bhttp for Gamma. --- src/scrapers/gamma.js | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index b08e5c28..975c2cc2 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -2,7 +2,6 @@ const Promise = require('bluebird'); const util = require('util'); -const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -27,7 +26,7 @@ function getAlbumUrl(albumPath, site) { } async function fetchPhotos(url) { - const res = await bhttp.get(url); + const res = await http.get(url); return res.body.toString(); } @@ -434,7 +433,7 @@ async function fetchApiLatest(site, page = 1, preData, include, upcoming = false const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`; const { apiUrl } = await fetchApiCredentials(referer, site); - const res = await bhttp.post(apiUrl, { + const res = await http.post(apiUrl, { requests: [ { indexName: 'all_scenes', @@ -442,9 +441,8 @@ async function fetchApiLatest(site, page = 1, preData, include, upcoming = false }, ], }, { - headers: { - Referer: referer, - }, + Referer: referer, + }, { encodeJSON: true, }); @@ -487,14 +485,14 @@ function getUpcomingUrl(site) { async function fetchLatest(site, page = 1) { const url = getLatestUrl(site, page); - const res = await bhttp.get(url); + const res = await http.get(url); return scrapeAll(res.body.toString(), site); } async function fetchUpcoming(site) { const url = getUpcomingUrl(site); - const res = await bhttp.get(url); + const res = await http.get(url); return scrapeAll(res.body.toString(), site, null, false); } @@ -533,12 +531,10 @@ async function fetchScene(url, site, baseRelease) { if (deepUrl) { const [res, mobileRes] = await Promise.all([ - bhttp.get(deepUrl), - mobileUrl && bhttp.get(mobileUrl, { - headers: { - // don't redirect to main site - 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36', - }, + http.get(deepUrl), + mobileUrl && http.get(mobileUrl, { + // don't redirect to main site + 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36', }), ]); @@ -554,7 +550,7 @@ async function fetchScene(url, site, baseRelease) { } async function fetchActorScenes(actorName, apiUrl, siteSlug) { - const res = await bhttp.post(apiUrl, { + const res = await http.post(apiUrl, { requests: [ { indexName: 'all_scenes', @@ -562,9 +558,8 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) { }, ], }, { - headers: { - Referer: `https://www.${siteSlug}.com/en/videos`, - }, + Referer: `https://www.${siteSlug}.com/en/videos`, + }, { encodeJSON: true, }); @@ -582,7 +577,7 @@ async function fetchProfile({ name: actorName }, context, altSearchUrl, getActor const searchUrl = altSearchUrl ? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor` : `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`; - const searchRes = await bhttp.get(searchUrl); + const searchRes = await http.get(searchUrl); if (searchRes.statusCode !== 200) { return null; @@ -592,7 +587,7 @@ async function fetchProfile({ name: actorName }, context, altSearchUrl, getActor if (actorUrl) { const url = `https://${siteSlug}.com${actorUrl}`; - const actorRes = await bhttp.get(url); + const actorRes = await http.get(url); if (actorRes.statusCode !== 200) { return null;