DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using new HTTP module with a dynamic rate limiter.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-22 04:07:09 +01:00
parent 5d0fe44130
commit b9b777c621
27 changed files with 358 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/scrapers/vixen.js
View File

@@ -4,7 +4,7 @@
const Promise = require('bluebird');
const moment = require('moment');
const { get, post } = require('../utils/http');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
const genderMap = {
@@ -45,13 +45,15 @@ function getAvatarFallbacks(avatar) {
async function getTrailer(scene, site, url) {
const qualities = [360, 480, 720, 1080, 2160];
const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
const tokenRes = await http.post(`${site.url}/api/__record_tknreq`, {
file: scene.previewVideoUrl1080P,
sizes: qualities.join('+'),
type: 'trailer',
}, {
referer: url,
origin: site.url,
headers: {
referer: url,
origin: site.url,
},
});
if (!tokenRes.ok) {
@@ -59,7 +61,7 @@ async function getTrailer(scene, site, url) {
}
const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
const trailersRes = await post(trailerUrl, null, { referer: url });
const trailersRes = await http.post(trailerUrl, null, { headers: { referer: url } });
if (trailersRes.ok) {
return qualities.map(quality => (trailersRes.body[quality] ? {
@@ -155,7 +157,7 @@ async function scrapeScene(data, url, site, baseRelease) {
async function fetchActorReleases(pages, model, origin) {
const releasesPerPage = await Promise.map(pages, async (page) => {
const url = `${origin}/api${model.targetUrl}?page=${page}`;
const res = await get(url);
const res = await http.get(url);
if (res.code === 200) {
return scrapeAll(res.body.data.videos.videos, null, origin);
@@ -203,7 +205,7 @@ async function scrapeProfile(data, origin, withReleases) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/api/videos?page=${page}`;
const res = await get(url);
const res = await http.get(url);
if (res.code === 200) {
return scrapeAll(res.body.data.videos, site);
@@ -214,7 +216,7 @@ async function fetchLatest(site, page = 1) {
async function fetchUpcoming(site) {
const apiUrl = `${site.url}/api`;
const res = await get(apiUrl);
const res = await http.get(apiUrl);
if (res.code === 200) {
return scrapeUpcoming(res.body.data.nextScene, site);
@@ -227,7 +229,7 @@ async function fetchScene(url, site, baseRelease) {
const { origin, pathname } = new URL(url);
const apiUrl = `${origin}/api${pathname}`;
const res = await get(apiUrl);
const res = await http.get(apiUrl);
if (res.code === 200) {
return scrapeScene(res.body.data, url, site, baseRelease);
@@ -240,7 +242,7 @@ async function fetchProfile({ name: actorName }, { site }, include) {
const origin = site.url;
const actorSlug = slugify(actorName);
const url = `${origin}/api/${actorSlug}`;
const res = await get(url);
const res = await http.get(url);
if (res.code === 200) {
return scrapeProfile(res.body.data, origin, include.scenes);