DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Using http instead of bhttp for Gamma.

This commit is contained in:
DebaucheryLibrarian 2020-09-10 17:41:29 +02:00
parent d8866172c1
commit 7f8704ee2c
1 changed files with 15 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/scrapers/gamma.js
View File

@ -2,7 +2,6 @@
const Promise = require('bluebird');
const util = require('util');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
@ -27,7 +26,7 @@ function getAlbumUrl(albumPath, site) {
}
async function fetchPhotos(url) {
const res = await bhttp.get(url);
const res = await http.get(url);
return res.body.toString();
}
@ -434,7 +433,7 @@ async function fetchApiLatest(site, page = 1, preData, include, upcoming = false
const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`;
const { apiUrl } = await fetchApiCredentials(referer, site);
const res = await bhttp.post(apiUrl, {
const res = await http.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@ -442,9 +441,8 @@ async function fetchApiLatest(site, page = 1, preData, include, upcoming = false
},
],
}, {
headers: {
Referer: referer,
},
Referer: referer,
}, {
encodeJSON: true,
});
@ -487,14 +485,14 @@ function getUpcomingUrl(site) {
async function fetchLatest(site, page = 1) {
const url = getLatestUrl(site, page);
const res = await bhttp.get(url);
const res = await http.get(url);
return scrapeAll(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const url = getUpcomingUrl(site);
const res = await bhttp.get(url);
const res = await http.get(url);
return scrapeAll(res.body.toString(), site, null, false);
}
@ -533,12 +531,10 @@ async function fetchScene(url, site, baseRelease) {
if (deepUrl) {
const [res, mobileRes] = await Promise.all([
bhttp.get(deepUrl),
mobileUrl && bhttp.get(mobileUrl, {
headers: {
// don't redirect to main site
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36',
},
http.get(deepUrl),
mobileUrl && http.get(mobileUrl, {
// don't redirect to main site
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Mobile Safari/537.36',
}),
]);
@ -554,7 +550,7 @@ async function fetchScene(url, site, baseRelease) {
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {
const res = await bhttp.post(apiUrl, {
const res = await http.post(apiUrl, {
requests: [
{
indexName: 'all_scenes',
@ -562,9 +558,8 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
},
],
}, {
headers: {
Referer: `https://www.${siteSlug}.com/en/videos`,
},
Referer: `https://www.${siteSlug}.com/en/videos`,
}, {
encodeJSON: true,
});
@ -582,7 +577,7 @@ async function fetchProfile({ name: actorName }, context, altSearchUrl, getActor
const searchUrl = altSearchUrl
? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor`
: `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`;
const searchRes = await bhttp.get(searchUrl);
const searchRes = await http.get(searchUrl);
if (searchRes.statusCode !== 200) {
return null;
@ -592,7 +587,7 @@ async function fetchProfile({ name: actorName }, context, altSearchUrl, getActor
if (actorUrl) {
const url = `https://${siteSlug}.com${actorUrl}`;
const actorRes = await bhttp.get(url);
const actorRes = await http.get(url);
if (actorRes.statusCode !== 200) {
return null;