assets/js/actors/actions.js

@@ -140,7 +140,6 @@ function initActorActions(store, _router) {
                             url
                             title
                             date
-                            slug
                             ${releaseActorsFragment}
                             ${releaseTagsFragment}
                             ${releasePosterFragment}

config/default.js

@@ -43,14 +43,6 @@ module.exports = {
         'burningangel',
         'brazzers',
         'milehighmedia',
-        [
-            'vixen',
-            'tushy',
-            'blacked',
-            'tushyraw',
-            'blackedraw',
-            'deeper',
-        ],
         [
             // Nubiles
             'nubiles',

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "traxxx",
-  "version": "1.99.0",
+  "version": "1.98.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.99.0",
+    "version": "1.98.1",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/actors.js

@@ -381,7 +381,7 @@ async function scrapeActors(actorNames) {
 
                         logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
 
-                        const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug, argv.withReleases);
+                        const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
 
                         if (profile) {
                             logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);

src/releases.js

@@ -153,7 +153,7 @@ function curateReleases(releases) {
 }
 
 async function attachChannelSite(release) {
-    if (!release.site?.isFallback) {
+    if (!release.site.isFallback) {
         return release;
     }
 

src/scrape-releases.js

@@ -45,13 +45,6 @@ async function scrapeRelease(source, basicRelease = null, type = 'scene', prefli
         throw new Error(`Could not find site ${url} in database`);
     }
 
-    if (!argv.deep && release) {
-        return {
-            ...release,
-            site,
-        };
-    }
-
     const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
 
     if (!scraper) {

src/scrapers/scrapers.js

@@ -114,15 +114,9 @@ module.exports = {
     },
     actors: {
         '21sextury': sextury,
-        analbbc: fullpornnetwork,
-        analized: fullpornnetwork,
-        analviolation: fullpornnetwork,
         anilos: nubiles,
         babes,
-        baddaddypov: fullpornnetwork,
         bangbros,
-        blacked: vixen,
-        blackedraw: vixen,
         blowpass,
         boobpedia,
         brattysis: nubiles,
@@ -130,47 +124,47 @@ module.exports = {
         burningangel,
         cherrypimps,
         ddfnetwork,
-        deeper: vixen,
         deeplush: nubiles,
         digitalplayground,
-        dtfsluts: fullpornnetwork,
         evilangel,
         fakehub,
         famedigital,
         freeones,
         freeonesLegacy,
-        girlfaction: fullpornnetwork,
-        hergape: fullpornnetwork,
-        homemadeanalwhores: fullpornnetwork,
         hotcrazymess: nubiles,
         iconmale,
-        jamesdeen: fullpornnetwork,
         julesjordan,
         kellymadison,
         legalporno,
         men,
+        analbbc: fullpornnetwork,
+        analized: fullpornnetwork,
+        analviolation: fullpornnetwork,
+        baddaddypov: fullpornnetwork,
+        dtfsluts: fullpornnetwork,
+        girlfaction: fullpornnetwork,
+        hergape: fullpornnetwork,
+        homemadeanalwhores: fullpornnetwork,
+        jamesdeen: fullpornnetwork,
+        mugfucked: fullpornnetwork,
+        onlyprince: fullpornnetwork,
+        pervertgallery: fullpornnetwork,
+        povperverts: fullpornnetwork,
         metrohd,
         milehighmedia,
         mofos,
-        mugfucked: fullpornnetwork,
         naughtyamerica,
         nfbusty: nubiles,
         nubilefilms: nubiles,
         nubiles,
         nubilesporn: nubiles,
-        onlyprince: fullpornnetwork,
-        pervertgallery: fullpornnetwork,
         pimpxxx: cherrypimps,
         pornhub,
-        povperverts: fullpornnetwork,
         realitykings,
         score,
         thatsitcomshow: nubiles,
         transangels,
-        tushy: vixen,
-        tushyraw: vixen,
         twistys,
-        vixen,
         wicked,
         xempire,
     },

src/scrapers/vixen.js

@@ -1,18 +1,10 @@
 'use strict';
 
 /* eslint-disable newline-per-chained-call */
-const Promise = require('bluebird');
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
 const moment = require('moment');
 
-const { get, post } = require('../utils/http');
-const slugify = require('../utils/slugify');
-
-const genderMap = {
-    F: 'female',
-    M: 'male',
-    T: 'transsexual', // not yet observed
-};
-
 function getPosterFallbacks(poster) {
     return poster
         .filter(image => /landscape/i.test(image.name))
@@ -25,46 +17,54 @@ function getPosterFallbacks(poster) {
         .flat();
 }
 
-function getTeaserFallbacks(teaser) {
+function scrapeLatest(html, site) {
-    return teaser
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
-        .filter(video => /landscape/i.test(video.name))
-        .map(video => ({
-            src: video.src,
-            type: video.type,
-            quality: Number(String(video.height).replace('353', '360')),
-        }));
-}
 
-function getAvatarFallbacks(avatar) {
+    const stateScript = $('script:contains("INITIAL_STATE")').html();
-    return avatar
+    const { videos: scenes } = JSON.parse(stateScript.slice(stateScript.indexOf('{'), stateScript.indexOf('};') + 1));
-        .sort((imageA, imageB) => imageB.height - imageA.height)
-        .map(image => [image.highdpi?.['3x'], image.highdpi?.['2x'], image.src])
-        .flat();
-}
 
-function scrapeAll(scenes, site, origin) {
     return scenes.map((scene) => {
-        const release = {};
+        const entryId = String(scene.newId);
 
-        release.title = scene.title;
+        const {
+            title,
+            models: actors,
+        } = scene;
 
-        release.entryId = String(scene.newId);
+        const url = `${site.url}${scene.targetUrl}`;
-        release.url = `${site?.url || origin}${scene.targetUrl}`;
+        const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
+        const stars = Number(scene.textRating) / 2;
 
-        release.date = moment.utc(scene.releaseDate).toDate();
+        // largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
-        release.shootDate = moment.utc(scene.shootDate).toDate();
+        const poster = scene.images.listing.slice(-1)[0].src;
+        const teaser = scene.previews.listing.slice(-1)[0];
 
-        release.actors = scene.models;
+        return {
-        release.stars = Number(scene.textRating) / 2;
+            url,
-
+            entryId,
-        release.poster = getPosterFallbacks(scene.images.poster);
+            title,
-        release.teaser = getTeaserFallbacks(scene.previews.poster);
+            actors,
-
+            date,
-        return release;
+            poster,
+            teaser: {
+                src: teaser.src,
+                type: teaser.type,
+                quality: teaser.height,
+            },
+            rating: {
+                stars,
+            },
+            site,
+        };
     });
 }
 
-function scrapeUpcoming(scene, site) {
+function scrapeUpcoming(html, site) {
+    const statePrefix = html.indexOf('__INITIAL_STATE__');
+    const stateString = html.slice(html.indexOf('{', statePrefix), html.indexOf('};', statePrefix) + 1);
+    const data = JSON.parse(stateString);
+
+    const scene = data.page.data['/'].data?.nextScene;
     if (!scene || scene.isPreReleasePeriod) return null;
 
     const release = {};
@@ -75,23 +75,33 @@ function scrapeUpcoming(scene, site) {
         .map(component => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
         .join(' ');
 
-    release.url = `${site.url}${scene.targetUrl}`;
-
     release.date = moment.utc(scene.releaseDate).toDate();
-    release.shootDate = moment.utc(scene.shootDate).toDate();
+    release.url = `${site.url}${scene.targetUrl}`;
 
     release.actors = scene.models;
 
     release.poster = getPosterFallbacks(scene.images.poster);
-    release.teaser = getTeaserFallbacks(scene.previews.poster);
+    release.teaser = scene.previews.poster
+        .filter(teaser => /landscape/i.test(teaser.name))
+        .map(teaser => ({
+            src: teaser.src,
+            type: teaser.type,
+            quality: Number(String(teaser.height).replace('353', '360')),
+        }));
 
     release.entryId = (release.poster[0] || release.teaser[0])?.match(/\/(\d+)/)?.[1];
 
     return [release];
 }
 
-async function scrapeScene(data, url, site, baseRelease) {
+async function scrapeScene(html, url) {
-    const scene = data.video;
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+
+    const stateObject = $('script:contains("INITIAL_STATE")');
+    const data = JSON.parse(stateObject.html().trim().slice(27, -1));
+
+    const pageData = data.page.data[data.location.pathname].data;
+    const scene = data.videos.find(video => video.newId === pageData.video);
 
     const release = {
         url,
@@ -104,132 +114,64 @@ async function scrapeScene(data, url, site, baseRelease) {
         tags: scene.tags,
     };
 
-    release.entryId = scene.newId;
+    release.entryId = pageData.video;
-
+    release.actors = scene.models;
-    release.date = moment.utc(scene.releaseDate).toDate();
-    release.shootDate = moment.utc(scene.shootDate).toDate();
-
-    release.actors = baseRelease?.actors || scene.models;
 
+    // release.poster = scene.rotatingThumbsUrlSizes[0]['1040w'];
     release.poster = getPosterFallbacks(scene.images.poster);
-    release.photos = data.pictureset.map(photo => photo.main[0].src);
+    release.photos = pageData.pictureset.map(photo => photo.main[0].src);
 
-    release.teaser = getTeaserFallbacks(scene.previews.poster);
+    const trailer = scene.previews.listing.find(preview => preview.height === 353);
+    if (trailer) release.trailer = { src: trailer };
 
-    const qualities = [360, 480, 720, 1080, 2160];
+    // trailer must exist!
-    const trailersUrl = `${site.url}/api/__tkn/${scene.previewVideoUrl1080P}/trailer/${qualities.join('+')}`;
-    const trailersRes = await post(trailersUrl, null, { headers: { referer: url } });
 
-    if (trailersRes.code === 200) {
+    release.teaser = scene.previews.poster
-        release.trailer = qualities.map(quality => (trailersRes.body[quality] ? {
+        .filter(teaser => /landscape/i.test(teaser.name))
-            src: trailersRes.body[quality].token,
+        .map(teaser => ({
-            quality,
+            src: teaser.src,
-        } : null)).filter(Boolean);
+            type: teaser.type,
-    }
+            quality: Number(String(teaser.height).replace('353', '360')),
+        }));
+
+    release.date = new Date(scene.releaseDate);
 
     return release;
 }
 
-async function fetchActorReleases(pages, model, origin) {
-    const releasesPerPage = await Promise.map(pages, async (page) => {
-        const url = `${origin}/api${model.targetUrl}?page=${page}`;
-        const res = await get(url);
-
-        if (res.code === 200) {
-            return scrapeAll(res.body.data.videos.videos, null, origin);
-        }
-
-        return [];
-    }, { concurrency: 3 });
-
-    return releasesPerPage.flat();
-}
-
-async function scrapeProfile(data, origin, withReleases) {
-    const model = data.model;
-    const profile = {};
-
-    profile.birthdate = new Date(model.dateOfBirth);
-    profile.gender = genderMap[model.sex];
-
-    profile.hair = model.hairColour;
-    profile.nationality = model.nationality;
-
-    if (model.biography.trim().length > 0) profile.description = model.biography;
-
-    if (model.cupSize && model.bustMeasurment) profile.bust = `${model.bustMeasurment}${model.cupSize}`;
-    if (model.waistMeasurment) profile.waist = model.waistMeasurment;
-    if (model.hipMeasurment) profile.hip = model.hipMeasurment;
-
-    profile.avatar = getAvatarFallbacks(model.images.listing);
-    profile.poster = getAvatarFallbacks(model.images.profile);
-    profile.banner = getAvatarFallbacks(model.images.poster);
-
-    const releases = scrapeAll(data.videos.videos, null, origin);
-
-    if (withReleases) {
-        const pageCount = Math.ceil(data.videos.count / 6);
-        const otherReleases = await fetchActorReleases((Array.from({ length: pageCount - 1 }, (value, index) => index + 2)), model, origin);
-
-        profile.releases = [...releases, ...otherReleases];
-    } else {
-        profile.releases = releases;
-    }
-
-    return profile;
-}
-
 async function fetchLatest(site, page = 1) {
-    const url = `${site.url}/api/videos?page=${page}`;
+    const url = `${site.url}/videos?page=${page}&size=7`;
-    const res = await get(url);
+    const res = await bhttp.get(url);
 
-    if (res.code === 200) {
+    if (res.statusCode === 200) {
-        return scrapeAll(res.body.data.videos, site);
+        return scrapeLatest(res.body.toString(), site);
     }
 
-    return res.code;
+    return res.statusCode;
 }
 
 async function fetchUpcoming(site) {
-    const apiUrl = `${site.url}/api`;
+    const res = await bhttp.get(site.url);
-    const res = await get(apiUrl);
 
-    if (res.code === 200) {
+    if (res.statusCode === 200) {
-        return scrapeUpcoming(res.body.data.nextScene, site);
+        return scrapeUpcoming(res.body.toString(), site);
     }
 
-    return res.code;
+    return res.statusCode;
 }
 
-async function fetchScene(url, site, baseRelease) {
+async function fetchScene(url, site) {
-    const { origin, pathname } = new URL(url);
+    const res = await bhttp.get(url);
-    const apiUrl = `${origin}/api${pathname}`;
 
-    const res = await get(apiUrl);
+    if (res.statusCode === 200) {
-
+        return scrapeScene(res.body.toString(), url, site);
-    if (res.code === 200) {
-        return scrapeScene(res.body.data, url, site, baseRelease);
     }
 
-    return res.code;
+    throw new Error(`Vixen response not OK for scene (${url}): ${res.statusCode}`);
-}
-
-async function fetchProfile(actorName, scraperSlug, withReleases) {
-    const origin = `https://www.${scraperSlug}.com`;
-    const actorSlug = slugify(actorName);
-    const url = `${origin}/api/${actorSlug}`;
-    const res = await get(url);
-
-    if (res.code === 200) {
-        return scrapeProfile(res.body.data, origin, withReleases);
-    }
-
-    return null;
 }
 
 module.exports = {
     fetchLatest,
     fetchUpcoming,
     fetchScene,
-    fetchProfile,
 };