DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using new HTTP module with a dynamic rate limiter.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-22 04:07:09 +01:00
parent 5d0fe44130
commit b9b777c621
27 changed files with 358 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/scrapers/kellymadison.js
View File

@@ -97,8 +97,10 @@ async function scrapeScene({ query, html }, url, baseRelease) {
const token = query.meta('name=_token');
const trailerInfoUrl = `${origin}/episodes/trailer/sources/${release.entryId}?type=trailer`;
const trailerInfoRes = await http.post(trailerInfoUrl, null, {
'X-CSRF-Token': token,
'X-Requested-With': 'XMLHttpRequest',
headers: {
'X-CSRF-Token': token,
'X-Requested-With': 'XMLHttpRequest',
},
});
if (trailerInfoRes.ok && trailerInfoRes.body.sources.length > 0) {
@@ -136,7 +138,9 @@ function scrapeProfile({ query }) {
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/episodes/search?page=${page}`; // TLS issues with teenfidelity.com, same overview on all sites
const res = await http.get(url, {
'X-Requested-With': 'XMLHttpRequest',
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok && res.body.status === 'success') {
@@ -157,7 +161,9 @@ async function fetchScene(url, channel, baseRelease) {
async function fetchProfile({ name: actorName }) {
const actorSlug = slugify(actorName);
const res = await qu.get(`https://www.kellymadison.com/models/${actorSlug}`, null, {
'X-Requested-With': 'XMLHttpRequest',
headers: {
'X-Requested-With': 'XMLHttpRequest',
},
});
if (res.ok) {