DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Using new HTTP module with a dynamic rate limiter.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-11-22 04:07:09 +01:00
parent 5d0fe44130
commit b9b777c621
27 changed files with 358 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/scrapers/gamma.js
View File

@@ -7,7 +7,7 @@ const cheerio = require('cheerio');
const moment = require('moment');
const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const qu = require('../utils/qu');
const http = require('../utils/http');
const slugify = require('../utils/slugify');
@@ -318,7 +318,7 @@ async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, acc
const profilePath = `/${pathname.split('/').slice(-2).join('/')}`;
const url = getActorReleasesUrl(profilePath, page);
const res = await get(url);
const res = await qu.get(url);
if (!res.ok) return [];
@@ -333,14 +333,14 @@ async function fetchActorReleases(profileUrl, getActorReleasesUrl, page = 1, acc
}
async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUrl, withReleases) {
const { q } = ex(html);
const { query } = qu.extract(html);
const avatar = q('img.actorPicture');
const hair = q('.actorProfile .attribute_hair_color', true);
const height = q('.actorProfile .attribute_height', true);
const weight = q('.actorProfile .attribute_weight', true);
const alias = q('.actorProfile .attribute_alternate_names', true);
const nationality = q('.actorProfile .attribute_home', true);
const avatar = query.el('img.actorPicture');
const hair = query.cnt('.actorProfile .attribute_hair_color');
const height = query.cnt('.actorProfile .attribute_height');
const weight = query.cnt('.actorProfile .attribute_weight');
const alias = query.cnt('.actorProfile .attribute_alternate_names');
const nationality = query.cnt('.actorProfile .attribute_home');
const profile = {
name: actorName,
@@ -358,7 +358,7 @@ async function scrapeProfile(html, url, actorName, _siteSlug, getActorReleasesUr
profile.avatar = avatars;
}
profile.description = q('.actorBio p:not(.bioTitle)', true);
profile.description = query.cnt('.actorBio p:not(.bioTitle)');
if (hair) profile.hair = hair.split(':')[1].trim();
if (height) profile.height = Number(height.match(/\d+/)[0]);