From 5c63bd860bb7e59e6221168d0be53590fd112abc Mon Sep 17 00:00:00 2001
From: Niels Simenon <niels.simenon@gmail.com>
Date: Fri, 6 Mar 2020 18:59:32 +0100
Subject: [PATCH] Tags module filters out undefined tags. Gamma will return []
 instead of undefined when no tags are available.

---
 seeds/01_networks.js  |  3 +++
 seeds/02_sites.js     | 26 +++++++++++++++++++++-----
 src/scrapers/gamma.js | 10 ++++------
 src/tags.js           |  2 +-
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/seeds/01_networks.js b/seeds/01_networks.js
index b00893ae..51fc898e 100644
--- a/seeds/01_networks.js
+++ b/seeds/01_networks.js
@@ -29,6 +29,9 @@ const networks = [
         name: '21Sextury',
         url: 'https://www.21sextury.com',
         description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
+        parameters: {
+            mobile: 'https://m.dpfanatics.com/en/video',
+        },
         parent: 'gamma',
     },
     {
diff --git a/seeds/02_sites.js b/seeds/02_sites.js
index 15a1e97b..0854f87c 100644
--- a/seeds/02_sites.js
+++ b/seeds/02_sites.js
@@ -156,7 +156,7 @@ const sites = [
         network: '21sextury',
         parameters: {
             referer: 'https://www.21sextury.com',
-            mobile: 'https://m.analteenangels.com/',
+            mobile: 'https://m.analteenangels.com/en/video',
         },
     },
     {
@@ -167,7 +167,7 @@ const sites = [
         network: '21sextury',
         parameters: {
             networkReferer: true,
-            mobile: 'https://m.assholefever.com/',
+            mobile: 'https://m.assholefever.com/en/video',
         },
     },
     {
@@ -177,6 +177,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -187,6 +188,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -197,7 +199,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
-            mobile: 'https://m.dpfanatics.com',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -209,7 +211,7 @@ const sites = [
         network: '21sextury',
         parameters: {
             referer: 'https://www.21sextury.com',
-            mobile: 'https://m.dpfanatics.com/',
+            mobile: 'https://m.dpfanatics.com/en/video',
         },
     },
     {
@@ -221,7 +223,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
-            mobile: 'https://m.footsiebabes.com',
+            mobile: 'https://m.footsiebabes.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -232,6 +234,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -243,6 +246,7 @@ const sites = [
         network: '21sextury',
         parameters: {
             referer: 'https://www.21sextury.com',
+            mobile: 'https://m.lezcuties.com/en/video',
         },
     },
     {
@@ -252,6 +256,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -262,6 +267,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -272,6 +278,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -282,6 +289,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -292,6 +300,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -302,6 +311,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -312,6 +322,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -322,6 +333,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -332,6 +344,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -342,6 +355,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -352,6 +366,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
@@ -362,6 +377,7 @@ const sites = [
         parameters: {
             referer: 'https://www.21sextury.com',
             scene: 'https://www.21sextury.com/en/video',
+            mobile: 'https://m.dpfanatics.com/en/video',
             photos: 'https://www.21sextury.com/en/photo',
         },
     },
diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js
index 6e6d6cc1..27f33f9a 100644
--- a/src/scrapers/gamma.js
+++ b/src/scrapers/gamma.js
@@ -229,7 +229,7 @@ async function scrapeScene(html, url, site, scrapedRelease, mobileHtml) {
     }
 
     const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
-    const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
+    const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
     release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
 
     const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim();
@@ -471,7 +471,7 @@ function getDeepUrl(url, site, release, mobile = false) {
     const pathname = release?.path || new URL(url).pathname;
 
     if (mobile) {
-        return `${site.parameters.mobile}${pathname}`;
+        return `${site.parameters.mobile}${pathname.replace('/en/video', '')}`;
     }
 
     if (site.parameters?.deep === 'network') {
@@ -479,7 +479,7 @@ function getDeepUrl(url, site, release, mobile = false) {
     }
 
     if (site.parameters?.deep) {
-        return `${site.parameters.deep}${pathname}`;
+        return `${site.parameters.deep}${pathname.replace('/en/video', '')}`;
     }
 
     return url;
@@ -493,8 +493,6 @@ async function fetchScene(url, site, release) {
     const deepUrl = getDeepUrl(url, site, release);
     const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true);
 
-    console.log(mobileUrl);
-
     if (deepUrl) {
         const [res, mobileRes] = await Promise.all([
             bhttp.get(deepUrl),
@@ -507,7 +505,7 @@ async function fetchScene(url, site, release) {
         ]);
 
         if (res.statusCode === 200) {
-            const mobileBody = mobileRes.statusCode === 200 ? mobileRes.body.toString() : null;
+            const mobileBody = mobileRes?.statusCode === 200 ? mobileRes.body.toString() : null;
             const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody);
             return { ...scene, deepUrl };
         }
diff --git a/src/tags.js b/src/tags.js
index 92dd7dd1..72dc75f3 100644
--- a/src/tags.js
+++ b/src/tags.js
@@ -62,7 +62,7 @@ async function matchTags(rawTags) {
 async function associateTags(release, releaseId) {
     const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || [];
 
-    const rawReleaseTags = release.tags || [];
+    const rawReleaseTags = release.tags.filter(Boolean) || [];
     const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string')
         ? await matchTags(release.tags) // scraper returned raw tags
         : rawReleaseTags; // tags already matched by (outdated) scraper