Added Vivid network. Added ASMR Fantasy to Adult Time. Storing deep URL in database. Added href to header links.

src/scrapers/adulttime.js

@@ -13,8 +13,8 @@ function curateRelease(release, site) {
     return release;
 }
 
-async function networkFetchScene(url, site) {
-    const scene = await fetchScene(url, site);
+async function networkFetchScene(url, site, release) {
+    const scene = await fetchScene(url, site, release);
 
     return curateRelease(scene, site);
 }

src/scrapers/gamma.js

@@ -184,7 +184,7 @@ async function scrapeScene(html, url, site, scrapedRelease) {
     const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
 
     const [data, data2] = json ? JSON.parse(json) : [];
-    const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
+    const videoData = videoJson && JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
 
     [release.entryId] = (scrapedRelease?.path || new URL(url).pathname).split('/').slice(-1);
     release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name;
@@ -217,14 +217,14 @@ async function scrapeScene(html, url, site, scrapedRelease) {
         }));
     }
 
-    const hasTrans = release.actors.some(actor => actor.gender === 'shemale');
+    const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
     const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
     release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
 
     const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim();
     if (channel) release.channel = slugify(channel, { delimiter: '' });
 
-    release.poster = videoData.picPreview;
+    if (videoData.picPreview && new URL(videoData.picPreview).pathname.length > 1) release.poster = videoData.picPreview; // sometimes links to just https://images02-fame.gammacdn.com/
 
     const photoLink = $('.picturesItem a').attr('href');
     if (photoLink) release.photos = await getPhotos(photoLink, site);
@@ -472,7 +472,9 @@ async function fetchScene(url, site, release) {
     const res = await bhttp.get(deepUrl);
 
     if (res.statusCode === 200) {
-        return scrapeScene(res.body.toString(), url, site, release);
+        const scene = await scrapeScene(res.body.toString(), url, site, release);
+
+        return { ...scene, deepUrl };
     }
 
     return null;

src/scrapers/puretaboo.js

@@ -2,7 +2,6 @@
 
 const { fetchApiLatest, fetchApiUpcoming, fetchScene } = require('./gamma');
 
-
 module.exports = {
     fetchLatest: fetchApiLatest,
     fetchScene,

src/scrapers/scrapers.js

@@ -44,6 +44,7 @@ const sextury = require('./21sextury');
 const teamskeet = require('./teamskeet');
 const transangels = require('./transangels');
 const twistys = require('./twistys');
+const vivid = require('./vivid');
 const vixen = require('./vixen');
 const vogov = require('./vogov');
 const wicked = require('./wicked');
@@ -90,6 +91,7 @@ module.exports = {
         score,
         teamskeet,
         twistys,
+        vivid,
         vixen,
         vogov,
         wicked,

src/scrapers/vivid.js

@@ -0,0 +1,114 @@
+'use strict';
+
+/* eslint-disable no-unused-vars */
+const bhttp = require('bhttp');
+
+const { get, date } = require('../utils/q');
+const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
+
+function scrapeLatestNative(scenes, site) {
+    return scenes.map((scene) => {
+        const release = {};
+
+        release.entryId = scene.id;
+        release.url = `${site.url}${scene.url}`;
+
+        release.title = scene.name;
+        release.date = date(scene.release_date, 'YYYY-MM-DD');
+        release.duration = parseInt(scene.runtime, 10) * 60;
+
+        release.actors = scene.cast?.map(actor => ({
+            name: actor.stagename,
+            gender: actor.gender.toLowerCase(),
+            avatar: actor.placard,
+        })) || [];
+
+        release.stars = Number(scene.rating);
+        release.poster = scene.placard_800 || scene.placard;
+
+        return release;
+    });
+}
+
+function scrapeSceneNative({ html, q, qa }, url, _site) {
+    const release = { url };
+
+    release.entryId = new URL(url).pathname.split('/')[2]; // eslint-disable-line prefer-destructuring
+
+    release.title = q('.scene-h2-heading', true);
+    release.description = q('.indie-model-p', true);
+
+    const dateString = qa('h5').find(el => /Released/.test(el.textContent)).textContent;
+    release.date = date(dateString, 'MMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
+
+    const duration = qa('h5').find(el => /Runtime/.test(el.textContent)).textContent;
+    const [hours, minutes] = duration.match(/\d+/g);
+
+    if (minutes) release.duration = (hours * 3600) + (minutes * 60);
+    else release.duration = hours * 60; // scene shorter that 1hr, hour match are minutes
+
+    release.actors = qa('h4 a[href*="/stars"], h4 a[href*="/celebs"]', true);
+    release.tags = qa('h5 a[href*="/categories"]', true);
+
+    const [poster, trailer] = html.match(/https:\/\/content.vivid.com(.*)(.jpg|.mp4)/g);
+    release.poster = poster;
+
+    if (trailer) {
+        release.trailer = {
+            src: trailer,
+        };
+    }
+
+    const channel = q('h5 a[href*="/sites"]', true);
+    if (channel) release.channel = channel.replace(/\.\w+/, '');
+
+    return release;
+}
+
+async function fetchLatestNative(site, page = 1) {
+    if (site.parameters?.useGamma) {
+        return fetchApiLatest(site, page);
+    }
+
+    const apiUrl = `${site.url}/videos/api/?limit=50&offset=${(page - 1) * 50}&sort=datedesc`;
+    const res = await bhttp.get(apiUrl, {
+        decodeJSON: true,
+    });
+
+    if (res.statusCode === 200 && res.body.code === 200) {
+        return scrapeLatestNative(res.body.responseData, site);
+    }
+
+    return null;
+}
+
+async function fetchUpcomingNative(site) {
+    if (site.parameters?.useGamma) {
+        return fetchApiUpcoming(site);
+    }
+
+    return null;
+}
+
+async function fetchSceneNative(url, site, release) {
+    if (site.parameters?.useGamma) {
+        return fetchScene(url, site, release);
+    }
+
+    const qScene = await get(url);
+
+    return qScene && scrapeSceneNative(qScene, url, site);
+}
+
+async function fetchSceneWrapper(url, site, release) {
+    const scene = await fetchScene(url, site, release);
+
+    return scene;
+}
+
+module.exports = {
+    fetchLatest: fetchApiLatest,
+    fetchProfile: fetchApiProfile,
+    fetchUpcoming: fetchApiUpcoming,
+    fetchScene: fetchSceneWrapper,
+};