Tags module filters out undefined tags. Gamma will return [] instead of undefined when no tags are available.

This commit is contained in:
ThePendulum 2020-03-06 18:59:32 +01:00
parent 3f30f80d34
commit 5c63bd860b
4 changed files with 29 additions and 12 deletions

View File

@ -29,6 +29,9 @@ const networks = [
name: '21Sextury', name: '21Sextury',
url: 'https://www.21sextury.com', url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.', description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parameters: {
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: 'gamma', parent: 'gamma',
}, },
{ {

View File

@ -156,7 +156,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.analteenangels.com/', mobile: 'https://m.analteenangels.com/en/video',
}, },
}, },
{ {
@ -167,7 +167,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
networkReferer: true, networkReferer: true,
mobile: 'https://m.assholefever.com/', mobile: 'https://m.assholefever.com/en/video',
}, },
}, },
{ {
@ -177,6 +177,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -187,6 +188,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -197,7 +199,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com', mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -209,7 +211,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.dpfanatics.com/', mobile: 'https://m.dpfanatics.com/en/video',
}, },
}, },
{ {
@ -221,7 +223,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.footsiebabes.com', mobile: 'https://m.footsiebabes.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -232,6 +234,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -243,6 +246,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.lezcuties.com/en/video',
}, },
}, },
{ {
@ -252,6 +256,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -262,6 +267,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -272,6 +278,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -282,6 +289,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -292,6 +300,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -302,6 +311,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -312,6 +322,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -322,6 +333,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -332,6 +344,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -342,6 +355,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -352,6 +366,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -362,6 +377,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },

View File

@ -229,7 +229,7 @@ async function scrapeScene(html, url, site, scrapedRelease, mobileHtml) {
} }
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale'); const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', '); const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim(); const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim();
@ -471,7 +471,7 @@ function getDeepUrl(url, site, release, mobile = false) {
const pathname = release?.path || new URL(url).pathname; const pathname = release?.path || new URL(url).pathname;
if (mobile) { if (mobile) {
return `${site.parameters.mobile}${pathname}`; return `${site.parameters.mobile}${pathname.replace('/en/video', '')}`;
} }
if (site.parameters?.deep === 'network') { if (site.parameters?.deep === 'network') {
@ -479,7 +479,7 @@ function getDeepUrl(url, site, release, mobile = false) {
} }
if (site.parameters?.deep) { if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname}`; return `${site.parameters.deep}${pathname.replace('/en/video', '')}`;
} }
return url; return url;
@ -493,8 +493,6 @@ async function fetchScene(url, site, release) {
const deepUrl = getDeepUrl(url, site, release); const deepUrl = getDeepUrl(url, site, release);
const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true); const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true);
console.log(mobileUrl);
if (deepUrl) { if (deepUrl) {
const [res, mobileRes] = await Promise.all([ const [res, mobileRes] = await Promise.all([
bhttp.get(deepUrl), bhttp.get(deepUrl),
@ -507,7 +505,7 @@ async function fetchScene(url, site, release) {
]); ]);
if (res.statusCode === 200) { if (res.statusCode === 200) {
const mobileBody = mobileRes.statusCode === 200 ? mobileRes.body.toString() : null; const mobileBody = mobileRes?.statusCode === 200 ? mobileRes.body.toString() : null;
const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody); const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody);
return { ...scene, deepUrl }; return { ...scene, deepUrl };
} }

View File

@ -62,7 +62,7 @@ async function matchTags(rawTags) {
async function associateTags(release, releaseId) { async function associateTags(release, releaseId) {
const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || []; const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || [];
const rawReleaseTags = release.tags || []; const rawReleaseTags = release.tags.filter(Boolean) || [];
const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string') const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string')
? await matchTags(release.tags) // scraper returned raw tags ? await matchTags(release.tags) // scraper returned raw tags
: rawReleaseTags; // tags already matched by (outdated) scraper : rawReleaseTags; // tags already matched by (outdated) scraper