DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Tags module filters out undefined tags. Gamma will return [] instead of undefined when no tags are available.

This commit is contained in:
ThePendulum 2020-03-06 18:59:32 +01:00
parent 3f30f80d34
commit 5c63bd860b
4 changed files with 29 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
seeds/01_networks.js
View File

@ -29,6 +29,9 @@ const networks = [
name: '21Sextury',
url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parameters: {
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: 'gamma',
},
{

26
seeds/02_sites.js
View File

@ -156,7 +156,7 @@ const sites = [
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
mobile: 'https://m.analteenangels.com/',
mobile: 'https://m.analteenangels.com/en/video',
},
},
{
@ -167,7 +167,7 @@ const sites = [
network: '21sextury',
parameters: {
networkReferer: true,
mobile: 'https://m.assholefever.com/',
mobile: 'https://m.assholefever.com/en/video',
},
},
{
@ -177,6 +177,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -187,6 +188,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -197,7 +199,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -209,7 +211,7 @@ const sites = [
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
mobile: 'https://m.dpfanatics.com/',
mobile: 'https://m.dpfanatics.com/en/video',
},
},
{
@ -221,7 +223,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.footsiebabes.com',
mobile: 'https://m.footsiebabes.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -232,6 +234,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -243,6 +246,7 @@ const sites = [
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
mobile: 'https://m.lezcuties.com/en/video',
},
},
{
@ -252,6 +256,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -262,6 +267,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -272,6 +278,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -282,6 +289,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -292,6 +300,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -302,6 +311,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -312,6 +322,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -322,6 +333,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -332,6 +344,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -342,6 +355,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -352,6 +366,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
@ -362,6 +377,7 @@ const sites = [
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},

10
src/scrapers/gamma.js
View File

@ -229,7 +229,7 @@ async function scrapeScene(html, url, site, scrapedRelease, mobileHtml) {
}
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || [];
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim();
@ -471,7 +471,7 @@ function getDeepUrl(url, site, release, mobile = false) {
const pathname = release?.path || new URL(url).pathname;
if (mobile) {
return `${site.parameters.mobile}${pathname}`;
return `${site.parameters.mobile}${pathname.replace('/en/video', '')}`;
}
if (site.parameters?.deep === 'network') {
@ -479,7 +479,7 @@ function getDeepUrl(url, site, release, mobile = false) {
}
if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname}`;
return `${site.parameters.deep}${pathname.replace('/en/video', '')}`;
}
return url;
@ -493,8 +493,6 @@ async function fetchScene(url, site, release) {
const deepUrl = getDeepUrl(url, site, release);
const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true);
console.log(mobileUrl);
if (deepUrl) {
const [res, mobileRes] = await Promise.all([
bhttp.get(deepUrl),
@ -507,7 +505,7 @@ async function fetchScene(url, site, release) {
]);
if (res.statusCode === 200) {
const mobileBody = mobileRes.statusCode === 200 ? mobileRes.body.toString() : null;
const mobileBody = mobileRes?.statusCode === 200 ? mobileRes.body.toString() : null;
const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody);
return { ...scene, deepUrl };
}

2
src/tags.js
View File

@ -62,7 +62,7 @@ async function matchTags(rawTags) {
async function associateTags(release, releaseId) {
const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || [];
const rawReleaseTags = release.tags || [];
const rawReleaseTags = release.tags.filter(Boolean) || [];
const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string')
? await matchTags(release.tags) // scraper returned raw tags
: rawReleaseTags; // tags already matched by (outdated) scraper