boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Removed direct bhttp usage from scrapers in favor of local http module. Deleted legacy scrapers, as old code is available via git repo history.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-23 00:05:02 +01:00
parent 3d427f7e1d
commit 0633197793
22 changed files with 77 additions and 537 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/scrapers/score.js
View File

@@ -1,9 +1,8 @@
'use strict';
const bhttp = require('bhttp');
const { ex, exa, get } = require('../utils/q');
const slugify = require('../utils/slugify');
const http = require('../utils/http');
const { heightToCm, lbsToKg } = require('../utils/convert');
function scrapePhotos(html) {
@@ -19,7 +18,7 @@ function scrapePhotos(html) {
}
async function fetchPhotos(url) {
const res = await bhttp.get(url);
const res = await http.get(url);
if (res.statusCode === 200) {
return scrapePhotos(res.body.toString(), url);
@@ -198,7 +197,7 @@ async function scrapeProfile(html, actorUrl, withReleases) {
async function fetchLatest(site, page = 1) {
const latestPath = site.parameters?.path || '/big-boob-videos';
const url = `${site.url}${latestPath}?page=${page}`;
const res = await bhttp.get(url);
const res = await http.get(url);
if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site);
@@ -208,7 +207,7 @@ async function fetchLatest(site, page = 1) {
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await http.get(url);
if (res.statusCode === 200) {
return scrapeScene(res.body.toString(), url, site);
@@ -227,7 +226,7 @@ async function fetchProfile({ name: actorName }, context, include, page = 1, sou
const url = sources[source];
const res = await bhttp.get(url, {
const res = await http.get(url, {
followRedirects: false,
});
@@ -235,7 +234,7 @@ async function fetchProfile({ name: actorName }, context, include, page = 1, sou
const actorUrl = scrapeModels(res.body.toString(), actorName);
if (actorUrl) {
const actorRes = await bhttp.get(actorUrl);
const actorRes = await http.get(actorUrl);
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString(), actorUrl, include.scenes);