Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions
--- a/src/scrapers/brazzers.js
+++ b/src/scrapers/brazzers.js
@@ -93,8 +93,8 @@ async function scrapeScene(html, url, site) {
    };
 }

-async function fetchLatest(site) {
-    const res = await bhttp.get(`${site.url}/`);
+async function fetchLatest(site, page = 1) {
+    const res = await bhttp.get(`${site.url}/page/${page}/`);

    return scrape(res.body.toString(), site, false);
 }