DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

Browse Source
This commit is contained in:
ThePendulum
2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/scrapers/xempire.js
View File

@@ -121,15 +121,10 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site, storedReleases) {
const res = await bhttp.get(`${site.url}/en/videos`);
const releases = scrape(res.body.toString(), site);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
const storedShootIds = new Set(storedReleases.map(release => release.shoot_id));
const newReleases = releases.filter(release => !storedShootIds.has(release.shootId));
console.log(newReleases);
return scrape(res.body.toString(), site);
}
async function fetchUpcoming(site) {