From 5e461f1f1a5a48499af198a016af53a43ed85b9f Mon Sep 17 00:00:00 2001
From: Niels Simenon <niels.simenon@gmail.com>
Date: Thu, 13 Feb 2020 23:05:28 +0100
Subject: [PATCH] Added proper Gamma parameters to 21Sextury sites.

---
 seeds/02_sites.js      | 112 +++++++++++++++++++++++++++++++++++++----
 src/scrape-releases.js |   8 +--
 src/scrape-sites.js    |   2 +-
 src/scrapers/gamma.js  |  29 +++++++----
 4 files changed, 128 insertions(+), 23 deletions(-)

diff --git a/seeds/02_sites.js b/seeds/02_sites.js
index fe174738..ae19aade 100644
--- a/seeds/02_sites.js
+++ b/seeds/02_sites.js
@@ -154,6 +154,9 @@ const sites = [
         url: 'https://www.analteenangels.com',
         description: 'AnalTeenAngels is presented by the 21Sextury nextwork and features young, European teens in hardcore anal porn. Watch these barely legal teens have their first anal sex and give up their ass for some anal pounding!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'assholefever',
@@ -167,16 +170,31 @@ const sites = [
         slug: 'buttplays',
         name: 'Butt Plays',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'clubsandy',
         name: 'Club Sandy',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'deepthroatfrenzy',
         name: 'Deepthroat Frenzy',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'dpfanatics',
@@ -184,6 +202,9 @@ const sites = [
         url: 'https://www.dpfanatics.com',
         description: 'Welcome to DPFanatics, brought to you by 21Sextury. DP Fanatics brings you the best DP sex and double penetration porn you can find. Double vaginal penetration, double anal, amateur and teen DP inside!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'footsiebabes',
@@ -191,11 +212,21 @@ const sites = [
         url: 'https://www.footsiebabes.com',
         description: 'Welcome to FootsieBabes.com, bringing you the best foot porn, teen feet and foot worship you can find on the net. Watch stocking porn, footjobs, feet tickling and more inside!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'gapeland',
         name: 'Gapeland',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'lezcuties',
@@ -203,66 +234,129 @@ const sites = [
         url: 'https://www.lezcuties.com',
         description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'onlyswallows',
         name: 'Only Swallows',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'alettaoceanempire',
         name: 'Aletta Ocean Empire',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'analqueenalysa',
         name: 'Anal Queen Alysa',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'blueangellive',
         name: 'Blue Angel Live',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'pixandvideo',
         name: 'Pix and Video',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'cheatingwhorewives',
         name: 'Cheating Whore Wives',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'cutiesgalore',
         name: 'Cuties Galore',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'hotmilfclub',
         name: 'Hot MILF Club',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'letsplaylez',
         name: 'Lets Play Lez',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'nudefightclub',
         name: 'Nude Fight Club',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'sexwithkathianobili',
         name: 'Sex With Kathia Nobili',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'sweetsophiemoone',
         name: 'Sweet Sophie Moone',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     // ADULT TIME
     {
@@ -272,7 +366,7 @@ const sites = [
         network: 'adulttime',
         parameters: {
             referer: 'https://freetour.adulttime.com/en/join',
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
             scene: false,
         },
     },
@@ -283,7 +377,7 @@ const sites = [
         network: 'adulttime',
         parameters: {
             referer: 'https://freetour.bubblegumdungeon.com/en/join',
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
             scene: false,
         },
     },
@@ -1685,7 +1779,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1697,7 +1791,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1709,7 +1803,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1721,7 +1815,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1733,7 +1827,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1745,7 +1839,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     // GIRLSWAY
@@ -1881,7 +1975,7 @@ const sites = [
             referer: 'https://freetour.adulttime.com/en/blackforwife',
             useGamma: true,
             scene: false,
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
         },
     },
     // JULES JORDAN
diff --git a/src/scrape-releases.js b/src/scrape-releases.js
index 4164fbe5..53208abe 100644
--- a/src/scrape-releases.js
+++ b/src/scrape-releases.js
@@ -35,11 +35,11 @@ async function findSite(url, release) {
 
 async function scrapeRelease(source, basicRelease = null, type = 'scene', preflight) {
     // profile scraper may return either URLs or pre-scraped scenes
-    const sourceIsUrl = typeof source === 'string';
-    const url = sourceIsUrl ? source : source?.url;
-    const release = sourceIsUrl ? basicRelease : source;
+    const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
+    const url = sourceIsUrlOrEmpty ? source : source?.url;
+    const release = sourceIsUrlOrEmpty ? basicRelease : source;
 
-    const site = await findSite(url, release);
+    const site = basicRelease?.site || await findSite(url, release);
 
     if (!site) {
         throw new Error(`Could not find site ${url} in database`);
diff --git a/src/scrape-sites.js b/src/scrape-sites.js
index 6f1fc704..b0ab2d67 100644
--- a/src/scrape-sites.js
+++ b/src/scrape-sites.js
@@ -98,7 +98,7 @@ async function deepFetchReleases(baseReleases, preflight) {
     return Promise.map(baseReleases, async (release) => {
         if (release.url || (release.path && release.site)) {
             try {
-                const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene', preflight);
+                const fullRelease = await scrapeRelease(release.url, release, 'scene', preflight);
 
                 if (fullRelease) {
                     return {
diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js
index 180dd22a..0fad7c62 100644
--- a/src/scrapers/gamma.js
+++ b/src/scrapers/gamma.js
@@ -14,11 +14,15 @@ const slugify = require('../utils/slugify');
 function getAlbumUrl(albumPath, site) {
     if (site.parameters?.photos) {
         return /^http/.test(site.parameters.photos)
-            ? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
-            : `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
+            ? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
+            : `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
     }
 
-    return `${site.url}${albumPath}`;
+    if (site.url) {
+        return `${site.url}${albumPath}`;
+    }
+
+    return null;
 }
 
 async function fetchPhotos(url) {
@@ -59,6 +63,10 @@ function scrapePhotos(html) {
 async function getPhotos(albumPath, site) {
     const albumUrl = getAlbumUrl(albumPath, site);
 
+    if (!albumUrl) {
+        return [];
+    }
+
     try {
         const html = await fetchPhotos(albumUrl);
         const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -70,7 +78,7 @@ async function getPhotos(albumPath, site) {
             const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
 
             const otherPhotos = await Promise.map(otherPages, async (page) => {
-                const pageUrl = `${site.url}/${albumPath}/${page}`;
+                const pageUrl = `${albumUrl}/${page}`;
                 const pageHtml = await fetchPhotos(pageUrl);
 
                 return scrapePhotos(pageHtml, site);
@@ -107,7 +115,7 @@ async function scrapeApiReleases(json, site) {
         release.path = `/${scene.url_title}/${release.entryId}`;
 
         if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
-        else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
+        else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
 
         release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
         release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
@@ -469,12 +477,15 @@ async function fetchScene(url, site, release) {
     }
 
     const deepUrl = getDeepUrl(url, site, release);
-    const res = await bhttp.get(deepUrl);
 
-    if (res.statusCode === 200) {
-        const scene = await scrapeScene(res.body.toString(), url, site, release);
+    if (deepUrl) {
+        const res = await bhttp.get(deepUrl);
 
-        return { ...scene, deepUrl };
+        if (res.statusCode === 200) {
+            const scene = await scrapeScene(res.body.toString(), url, site, release);
+
+            return { ...scene, deepUrl };
+        }
     }
 
     return null;