Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

This commit is contained in:
2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions

View File

@@ -107,8 +107,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site, _date) {
const res = await bhttp.get(`${site.url}/trial/categories/movies_1_d.html`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/trial/categories/movies_${page}_d.html`);
return scrapeLatest(res.body.toString(), site);
}