From 054dfba6b5566518eb5e1b800099261a936f5311 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Thu, 6 Feb 2020 00:22:31 +0100 Subject: [PATCH] Fixed Score module to scrape releases for actors with just one page. --- src/scrapers/score.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/scrapers/score.js b/src/scrapers/score.js index 499da36f..544b40e2 100644 --- a/src/scrapers/score.js +++ b/src/scrapers/score.js @@ -139,7 +139,7 @@ async function fetchActorReleases(url, accReleases = []) { const releases = accReleases.concat(scrapeAll(document.body.outerHTML)); const nextPage = qu('.next-pg'); - if (nextPage && new URL(nextPage).searchParams.has('page')) { + if (nextPage && new URL(nextPage).searchParams.has('page')) { // last page has 'next' button linking to join page return fetchActorReleases(nextPage, releases); } @@ -149,7 +149,7 @@ async function fetchActorReleases(url, accReleases = []) { return null; } -async function scrapeProfile(html) { +async function scrapeProfile(html, actorUrl) { const { q, qa, qi } = ex(html, '#model-page'); const profile = { gender: 'female' }; @@ -187,10 +187,8 @@ async function scrapeProfile(html) { const avatar = qi('img'); if (avatar) profile.avatar = avatar; - const { qu } = ex(html, '#model-page + .container, #model-page + .container-fluid'); - const releasesPage = qu('.next-pg'); - - if (releasesPage) profile.releases = await fetchActorReleases(releasesPage); + const { origin, pathname } = new URL(actorUrl); + profile.releases = await fetchActorReleases(`${origin}${pathname}/scenes?page=1`); return profile; } @@ -238,7 +236,7 @@ async function fetchProfile(actorName, scraperSlug, page = 1, source = 0) { const actorRes = await bhttp.get(actorUrl); if (actorRes.statusCode === 200) { - return scrapeProfile(actorRes.body.toString()); + return scrapeProfile(actorRes.body.toString(), actorUrl); } return null;