Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions
--- a/src/scrapers/private.js
+++ b/src/scrapers/private.js
@@ -85,8 +85,8 @@ async function scrapeScene(html, url, site) {
    };
 }

-async function fetchLatest(site) {
-    const res = await bhttp.get(`${site.url}/`);
+async function fetchLatest(site, page = 1) {
+    const res = await bhttp.get(`${site.url}/${page}/`);

    return scrapeLatest(res.body.toString(), site);
 }