Fetching old scene links from Vivid search API.

seeds/01_sites.js

@@ -4454,36 +4454,48 @@ const sites = [
         name: 'Vivid Celeb',
         url: 'https://www.vividceleb.com',
         network: 'vivid',
+        parameters: {
+            referer: 'https://www.thebrats.com',
+            deep: 'https://www.thebrats.com/en/video',
+            scene: false,
+            lastNative: new Date('2018-03-25'),
+        },
     },
     {
         slug: 'thebrats',
         name: 'The Brats',
         url: 'https://www.thebrats.com',
         network: 'vivid',
-        parameters: {
-            useGamma: true,
-        },
     },
     {
         slug: 'wheretheboysarent',
         name: 'Where The Boys Aren\'t',
         url: 'https://www.wheretheboysarent.com',
         network: 'vivid',
-        parameters: {
-            useGamma: true,
-        },
     },
     {
         slug: 'nineteen',
         name: 'Nineteen',
         url: 'https://www.nineteen.com',
         network: 'vivid',
+        parameters: {
+            referer: 'https://www.thebrats.com',
+            deep: 'https://www.thebrats.com/en/video',
+            scene: false,
+            lastNative: new Date('2019-01-23'),
+        },
     },
     {
         slug: 'nastystepfamily',
         name: 'Nasty Step Family',
         url: 'https://www.nastystepfamily.com',
         network: 'vivid',
+        parameters: {
+            referer: 'https://www.thebrats.com',
+            deep: 'https://www.thebrats.com/en/video',
+            scene: false,
+            lastNative: new Date('2019-01-29'),
+        },
     },
     {
         slug: 'girlswhofuckgirls',
@@ -4491,10 +4503,10 @@ const sites = [
         url: 'https://www.girlswhofuckgirls.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-05-21'),
         },
     },
     {
@@ -4503,10 +4515,10 @@ const sites = [
         url: 'https://www.petited.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-01-28'),
         },
     },
     {
@@ -4515,10 +4527,10 @@ const sites = [
         url: 'https://www.orgytrain.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-01-09'),
         },
     },
     {
@@ -4527,10 +4539,10 @@ const sites = [
         url: 'https://www.momisamilf.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-01-25'),
         },
     },
     {
@@ -4539,10 +4551,10 @@ const sites = [
         url: 'https://www.blackwhitefuckfest.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-01-30'),
         },
     },
     {
@@ -4551,10 +4563,10 @@ const sites = [
         url: 'https://www.65inchhugeasses.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2019-05-18'),
         },
     },
     {
@@ -4563,10 +4575,10 @@ const sites = [
         url: 'https://www.brandnewfaces.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2018-02-28'),
         },
     },
     {
@@ -4575,10 +4587,10 @@ const sites = [
         url: 'https://www.vividclassic.com',
         network: 'vivid',
         parameters: {
-            useGamma: true,
             referer: 'https://www.thebrats.com',
             deep: 'https://www.thebrats.com/en/video',
             scene: false,
+            lastNative: new Date('2016-06-29'),
         },
     },
     // VIXEN

src/scrapers/vivid.js

@@ -5,6 +5,7 @@ const bhttp = require('bhttp');
 
 const { get, date } = require('../utils/q');
 const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
+const slugify = require('../utils/slugify');
 
 function scrapeLatestNative(scenes, site) {
     return scenes.map((scene) => {
@@ -103,6 +104,26 @@ async function fetchSceneNative(url, site, release) {
 async function fetchSceneWrapper(url, site, release) {
     const scene = await fetchScene(url, site, release);
 
+    if (scene.date - new Date(site.parameters.lastNative) <= 0) {
+        // scene is probably still available on Vivid site, use search API to get URL and original date
+        const searchUrl = `${site.url}/videos/api/?limit=10&sort=datedesc&search=${encodeURI(scene.title)}`;
+        const searchRes = await bhttp.get(searchUrl, {
+            decodeJSON: true,
+        });
+
+        if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
+            const sceneMatch = searchRes.body.responseData.find(item => slugify(item.name) === slugify(scene.title));
+
+            if (sceneMatch) {
+                return {
+                    ...scene,
+                    url: `${site.url}${sceneMatch.url}`,
+                    date: date(sceneMatch.release_date, 'YYYY-MM-DD'),
+                };
+            }
+        }
+    }
+
     return scene;
 }