DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Fixed Score module to scrape releases for actors with just one page.

This commit is contained in:
ThePendulum 2020-02-06 00:22:31 +01:00
parent ed1bc6c73f
commit 054dfba6b5
1 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/scrapers/score.js
View File

@ -139,7 +139,7 @@ async function fetchActorReleases(url, accReleases = []) {
const releases = accReleases.concat(scrapeAll(document.body.outerHTML)); const releases = accReleases.concat(scrapeAll(document.body.outerHTML));
const nextPage = qu('.next-pg'); const nextPage = qu('.next-pg');
if (nextPage && new URL(nextPage).searchParams.has('page')) { if (nextPage && new URL(nextPage).searchParams.has('page')) { // last page has 'next' button linking to join page
return fetchActorReleases(nextPage, releases); return fetchActorReleases(nextPage, releases);
} }
@ -149,7 +149,7 @@ async function fetchActorReleases(url, accReleases = []) {
return null; return null;
} }
async function scrapeProfile(html) { async function scrapeProfile(html, actorUrl) {
const { q, qa, qi } = ex(html, '#model-page'); const { q, qa, qi } = ex(html, '#model-page');
const profile = { gender: 'female' }; const profile = { gender: 'female' };
@ -187,10 +187,8 @@ async function scrapeProfile(html) {
const avatar = qi('img'); const avatar = qi('img');
if (avatar) profile.avatar = avatar; if (avatar) profile.avatar = avatar;
const { qu } = ex(html, '#model-page + .container, #model-page + .container-fluid'); const { origin, pathname } = new URL(actorUrl);
const releasesPage = qu('.next-pg'); profile.releases = await fetchActorReleases(`${origin}${pathname}/scenes?page=1`);
if (releasesPage) profile.releases = await fetchActorReleases(releasesPage);
return profile; return profile;
} }
@ -238,7 +236,7 @@ async function fetchProfile(actorName, scraperSlug, page = 1, source = 0) {
const actorRes = await bhttp.get(actorUrl); const actorRes = await bhttp.get(actorUrl);
if (actorRes.statusCode === 200) { if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString()); return scrapeProfile(actorRes.body.toString(), actorUrl);
} }
return null; return null;