Scraping upcoming Vixen scenes. Fetching release media groups sequentially to prevent collisions.

src/scrapers/vixen.js

@@ -5,17 +5,6 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const { matchTags } = require('../tags');
-
-const defaultTags = {
-    blacked: ['bbc'],
-    blackedraw: ['bbc'],
-    tushy: ['anal'],
-    tushyraw: ['anal'],
-    vixen: [],
-    deeper: [],
-};
-
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
@@ -36,7 +25,7 @@ function scrapeLatest(html, site) {
 
         // largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
         const poster = scene.images.listing.slice(-1)[0].src;
-        const trailer = scene.previews.listing.slice(-1)[0];
+        const teaser = scene.previews.listing.slice(-1)[0];
 
         return {
             url,
@@ -45,10 +34,10 @@ function scrapeLatest(html, site) {
             actors,
             date,
             poster,
-            trailer: {
-                src: trailer.src,
-                type: trailer.type,
-                quality: trailer.height,
+            teaser: {
+                src: teaser.src,
+                type: teaser.type,
+                quality: teaser.height,
             },
             rating: {
                 stars,
@@ -58,6 +47,50 @@ function scrapeLatest(html, site) {
     });
 }
 
+function scrapeUpcoming(html, site) {
+    const statePrefix = html.indexOf('__INITIAL_STATE__');
+    const stateString = html.slice(html.indexOf('{', statePrefix), html.indexOf('};', statePrefix) + 1);
+    const data = JSON.parse(stateString);
+
+    const scene = data.page.data['/'].data?.nextScene;
+    if (!scene) return null;
+
+    const release = {};
+
+    release.title = scene.targetUrl
+        .slice(1)
+        .split('-')
+        .map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
+        .join(' ');
+
+    release.date = moment.utc(scene.releaseDate).toDate();
+    release.url = `${site.url}${scene.targetUrl}`;
+
+    release.actors = scene.models;
+
+    release.poster = scene.images.poster
+        .filter(image => /landscape/i.test(image.name))
+        .sort((imageA, imageB) => imageB.height - imageA.height)
+        .map((image) => {
+            const sources = [image.src, image.highdpi?.['2x'], image.highdpi?.['3x']];
+            // high DPI images for full HD source are huge, only prefer for smaller fallback sources
+            return image.height === 1080 ? sources : sources.reverse();
+        })
+        .flat();
+
+    release.teaser = scene.previews.poster
+        .filter(teaser => /landscape/i.test(teaser.name))
+        .map(teaser => ({
+            src: teaser.src,
+            type: teaser.type,
+            quality: Number(String(teaser.height).replace('353', '360')),
+        }));
+
+    release.entryId = (release.poster[0] || release.teaser[0])?.match(/\/(\d+)/)?.[1];
+
+    return [release];
+}
+
 async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
@@ -68,6 +101,8 @@ async function scrapeScene(html, url, site) {
     const entryId = pageData.video;
     const scene = data.videos.find(video => video.newId === entryId);
 
+    console.log(scene, data, pageData);
+
     const poster = scene.rotatingThumbsUrlSizes[0]['1040w'];
     const photos = pageData.pictureset.map(photo => photo.main[0].src);
     const trailer = scene.previews.listing.find(preview => preview.height === 353) || null;
@@ -79,11 +114,10 @@ async function scrapeScene(html, url, site) {
         totalRateVal: stars,
         runLength: duration,
         directorNames: director,
-        tags: rawTags,
+        tags,
     } = scene;
 
     const date = new Date(scene.releaseDate);
-    const tags = await matchTags([...defaultTags[site.slug], ...rawTags]);
 
     return {
         url,
@@ -117,7 +151,17 @@ async function fetchLatest(site, page = 1) {
         return scrapeLatest(res.body.toString(), site);
     }
 
-    throw new Error(`Vixen response not OK for latest: ${res.statusCode}`);
+    return res.statusCode;
+}
+
+async function fetchUpcoming(site) {
+    const res = await bhttp.get(site.url);
+
+    if (res.statusCode === 200) {
+        return scrapeUpcoming(res.body.toString(), site);
+    }
+
+    return res.statusCode;
 }
 
 async function fetchScene(url, site) {
@@ -132,5 +176,6 @@ async function fetchScene(url, site) {
 
 module.exports = {
     fetchLatest,
+    fetchUpcoming,
     fetchScene,
 };