Added proper Gamma parameters to 21Sextury sites.

seeds/02_sites.js

@@ -154,6 +154,9 @@ const sites = [
         url: 'https://www.analteenangels.com',
         description: 'AnalTeenAngels is presented by the 21Sextury nextwork and features young, European teens in hardcore anal porn. Watch these barely legal teens have their first anal sex and give up their ass for some anal pounding!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'assholefever',
@@ -167,16 +170,31 @@ const sites = [
         slug: 'buttplays',
         name: 'Butt Plays',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'clubsandy',
         name: 'Club Sandy',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'deepthroatfrenzy',
         name: 'Deepthroat Frenzy',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'dpfanatics',
@@ -184,6 +202,9 @@ const sites = [
         url: 'https://www.dpfanatics.com',
         description: 'Welcome to DPFanatics, brought to you by 21Sextury. DP Fanatics brings you the best DP sex and double penetration porn you can find. Double vaginal penetration, double anal, amateur and teen DP inside!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'footsiebabes',
@@ -191,11 +212,21 @@ const sites = [
         url: 'https://www.footsiebabes.com',
         description: 'Welcome to FootsieBabes.com, bringing you the best foot porn, teen feet and foot worship you can find on the net. Watch stocking porn, footjobs, feet tickling and more inside!',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'gapeland',
         name: 'Gapeland',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'lezcuties',
@@ -203,66 +234,129 @@ const sites = [
         url: 'https://www.lezcuties.com',
         description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+        },
     },
     {
         slug: 'onlyswallows',
         name: 'Only Swallows',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'alettaoceanempire',
         name: 'Aletta Ocean Empire',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'analqueenalysa',
         name: 'Anal Queen Alysa',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'blueangellive',
         name: 'Blue Angel Live',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'pixandvideo',
         name: 'Pix and Video',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'cheatingwhorewives',
         name: 'Cheating Whore Wives',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'cutiesgalore',
         name: 'Cuties Galore',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'hotmilfclub',
         name: 'Hot MILF Club',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'letsplaylez',
         name: 'Lets Play Lez',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'nudefightclub',
         name: 'Nude Fight Club',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'sexwithkathianobili',
         name: 'Sex With Kathia Nobili',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     {
         slug: 'sweetsophiemoone',
         name: 'Sweet Sophie Moone',
         network: '21sextury',
+        parameters: {
+            referer: 'https://www.21sextury.com',
+            scene: 'https://www.21sextury.com/en/video',
+            photos: 'https://www.21sextury.com/en/photo',
+        },
     },
     // ADULT TIME
     {
@@ -272,7 +366,7 @@ const sites = [
         network: 'adulttime',
         parameters: {
             referer: 'https://freetour.adulttime.com/en/join',
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
             scene: false,
         },
     },
@@ -283,7 +377,7 @@ const sites = [
         network: 'adulttime',
         parameters: {
             referer: 'https://freetour.bubblegumdungeon.com/en/join',
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
             scene: false,
         },
     },
@@ -1685,7 +1779,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1697,7 +1791,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1709,7 +1803,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1721,7 +1815,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1733,7 +1827,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     {
@@ -1745,7 +1839,7 @@ const sites = [
             latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/',
             upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/',
             deep: 'network',
-            photos: 'https://www.fantasymassage.com/en/photo/',
+            photos: 'https://www.fantasymassage.com/en/photo',
         },
     },
     // GIRLSWAY
@@ -1881,7 +1975,7 @@ const sites = [
             referer: 'https://freetour.adulttime.com/en/blackforwife',
             useGamma: true,
             scene: false,
-            deep: 'https://21sextury.com/en/video/',
+            deep: 'https://21sextury.com/en/video',
         },
     },
     // JULES JORDAN

src/scrape-releases.js

@@ -35,11 +35,11 @@ async function findSite(url, release) {
 
 async function scrapeRelease(source, basicRelease = null, type = 'scene', preflight) {
     // profile scraper may return either URLs or pre-scraped scenes
-    const sourceIsUrl = typeof source === 'string';
+    const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
-    const url = sourceIsUrl ? source : source?.url;
+    const url = sourceIsUrlOrEmpty ? source : source?.url;
-    const release = sourceIsUrl ? basicRelease : source;
+    const release = sourceIsUrlOrEmpty ? basicRelease : source;
 
-    const site = await findSite(url, release);
+    const site = basicRelease?.site || await findSite(url, release);
 
     if (!site) {
         throw new Error(`Could not find site ${url} in database`);

src/scrape-sites.js

@@ -98,7 +98,7 @@ async function deepFetchReleases(baseReleases, preflight) {
     return Promise.map(baseReleases, async (release) => {
         if (release.url || (release.path && release.site)) {
             try {
-                const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene', preflight);
+                const fullRelease = await scrapeRelease(release.url, release, 'scene', preflight);
 
                 if (fullRelease) {
                     return {

src/scrapers/gamma.js

@@ -14,13 +14,17 @@ const slugify = require('../utils/slugify');
 function getAlbumUrl(albumPath, site) {
     if (site.parameters?.photos) {
         return /^http/.test(site.parameters.photos)
-            ? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
+            ? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
-            : `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
+            : `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
     }
 
+    if (site.url) {
         return `${site.url}${albumPath}`;
     }
 
+    return null;
+}
+
 async function fetchPhotos(url) {
     const res = await bhttp.get(url);
 
@@ -59,6 +63,10 @@ function scrapePhotos(html) {
 async function getPhotos(albumPath, site) {
     const albumUrl = getAlbumUrl(albumPath, site);
 
+    if (!albumUrl) {
+        return [];
+    }
+
     try {
         const html = await fetchPhotos(albumUrl);
         const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -70,7 +78,7 @@ async function getPhotos(albumPath, site) {
             const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
 
             const otherPhotos = await Promise.map(otherPages, async (page) => {
-                const pageUrl = `${site.url}/${albumPath}/${page}`;
+                const pageUrl = `${albumUrl}/${page}`;
                 const pageHtml = await fetchPhotos(pageUrl);
 
                 return scrapePhotos(pageHtml, site);
@@ -107,7 +115,7 @@ async function scrapeApiReleases(json, site) {
         release.path = `/${scene.url_title}/${release.entryId}`;
 
         if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
-        else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
+        else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
 
         release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
         release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
@@ -469,6 +477,8 @@ async function fetchScene(url, site, release) {
     }
 
     const deepUrl = getDeepUrl(url, site, release);
+
+    if (deepUrl) {
         const res = await bhttp.get(deepUrl);
 
         if (res.statusCode === 200) {
@@ -476,6 +486,7 @@ async function fetchScene(url, site, release) {
 
             return { ...scene, deepUrl };
         }
+    }
 
     return null;
 }