Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

src/scrapers/vixen.js

@@ -14,7 +14,7 @@ function scrapeLatest(html, site) {
     const { videos: scenes } = JSON.parse(stateObject.html().trim().slice(27, -1));
 
     return scenes.map((scene) => {
-        const shootId = scene.newId;
+        const shootId = String(scene.newId);
         const title = scene.title;
         const url = `${site.url}${scene.targetUrl}`;
         const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
@@ -44,7 +44,7 @@ async function scrapeScene(html, url, site) {
 
     const scene = data.page.data[`${pathname}${search}`].data.video;
 
-    const shootId = scene.newId;
+    const shootId = String(scene.newId);
     const title = scene.title;
     const date = new Date(scene.releaseDate);
     const actors = scene.models;
@@ -72,8 +72,8 @@ async function scrapeScene(html, url, site) {
     };
 }
 
-async function fetchLatest(site) {
-    const res = await bhttp.get(`${site.url}/videos`);
+async function fetchLatest(site, page = 1) {
+    const res = await bhttp.get(`${site.url}/videos?page=${page}&size=7`);
 
     return scrapeLatest(res.body.toString(), site);
 }