DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Compare commits

base: DebaucheryLibrarian:19f46d4d86c0e896c1e9adbddbceea227cf45abc
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental
..
compare: DebaucheryLibrarian:3f30f80d348fa27dd06bb2de3fcfa1e734fa87a0
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental

No commits in common. "19f46d4d86c0e896c1e9adbddbceea227cf45abc" and "3f30f80d348fa27dd06bb2de3fcfa1e734fa87a0" have entirely different histories.
19f46d4d86 ... 3f30f80d34

6 changed files with 14 additions and 31 deletions
Show Stats Expand all files Collapse all files

2
package-lock.json generated
View File

@ -1,6 +1,6 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.107.1", "version": "1.107.0",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.107.1", "version": "1.107.0",
"description": "All the latest porn releases in one place", "description": "All the latest porn releases in one place",
"main": "src/app.js", "main": "src/app.js",
"scripts": { "scripts": {

3
seeds/01_networks.js
View File

@ -29,9 +29,6 @@ const networks = [
name: '21Sextury', name: '21Sextury',
url: 'https://www.21sextury.com', url: 'https://www.21sextury.com',
description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.', description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
parameters: {
mobile: 'https://m.dpfanatics.com/en/video',
},
parent: 'gamma', parent: 'gamma',
}, },
{ {

26
seeds/02_sites.js
View File

@ -156,7 +156,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.analteenangels.com/en/video', mobile: 'https://m.analteenangels.com/',
}, },
}, },
{ {
@ -167,7 +167,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
networkReferer: true, networkReferer: true,
mobile: 'https://m.assholefever.com/en/video', mobile: 'https://m.assholefever.com/',
}, },
}, },
{ {
@ -177,7 +177,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -188,7 +187,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -199,7 +197,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video', mobile: 'https://m.dpfanatics.com',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -211,7 +209,7 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.dpfanatics.com/en/video', mobile: 'https://m.dpfanatics.com/',
}, },
}, },
{ {
@ -223,7 +221,7 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.footsiebabes.com/en/video', mobile: 'https://m.footsiebabes.com',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -234,7 +232,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -246,7 +243,6 @@ const sites = [
network: '21sextury', network: '21sextury',
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
mobile: 'https://m.lezcuties.com/en/video',
}, },
}, },
{ {
@ -256,7 +252,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -267,7 +262,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -278,7 +272,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -289,7 +282,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -300,7 +292,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -311,7 +302,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -322,7 +312,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -333,7 +322,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -344,7 +332,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -355,7 +342,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -366,7 +352,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },
@ -377,7 +362,6 @@ const sites = [
parameters: { parameters: {
referer: 'https://www.21sextury.com', referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video', scene: 'https://www.21sextury.com/en/video',
mobile: 'https://m.dpfanatics.com/en/video',
photos: 'https://www.21sextury.com/en/photo', photos: 'https://www.21sextury.com/en/photo',
}, },
}, },

10
src/scrapers/gamma.js
View File

@ -229,7 +229,7 @@ async function scrapeScene(html, url, site, scrapedRelease, mobileHtml) {
} }
const hasTrans = release.actors?.some(actor => actor.gender === 'shemale'); const hasTrans = release.actors?.some(actor => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ') || []; const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim(); const channel = data?.productionCompany?.name || $('.studioLink a, .siteLink a').attr('title')?.trim();
@ -471,7 +471,7 @@ function getDeepUrl(url, site, release, mobile = false) {
const pathname = release?.path || new URL(url).pathname; const pathname = release?.path || new URL(url).pathname;
if (mobile) { if (mobile) {
return `${site.parameters.mobile}${pathname.replace('/en/video', '')}`; return `${site.parameters.mobile}${pathname}`;
} }
if (site.parameters?.deep === 'network') { if (site.parameters?.deep === 'network') {
@ -479,7 +479,7 @@ function getDeepUrl(url, site, release, mobile = false) {
} }
if (site.parameters?.deep) { if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname.replace('/en/video', '')}`; return `${site.parameters.deep}${pathname}`;
} }
return url; return url;
@ -493,6 +493,8 @@ async function fetchScene(url, site, release) {
const deepUrl = getDeepUrl(url, site, release); const deepUrl = getDeepUrl(url, site, release);
const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true); const mobileUrl = site.parameters?.mobile && getDeepUrl(url, site, release, true);
console.log(mobileUrl);
if (deepUrl) { if (deepUrl) {
const [res, mobileRes] = await Promise.all([ const [res, mobileRes] = await Promise.all([
bhttp.get(deepUrl), bhttp.get(deepUrl),
@ -505,7 +507,7 @@ async function fetchScene(url, site, release) {
]); ]);
if (res.statusCode === 200) { if (res.statusCode === 200) {
const mobileBody = mobileRes?.statusCode === 200 ? mobileRes.body.toString() : null; const mobileBody = mobileRes.statusCode === 200 ? mobileRes.body.toString() : null;
const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody); const scene = await scrapeScene(res.body.toString(), url, site, release, mobileBody);
return { ...scene, deepUrl }; return { ...scene, deepUrl };
} }

2
src/tags.js
View File

@ -62,7 +62,7 @@ async function matchTags(rawTags) {
async function associateTags(release, releaseId) { async function associateTags(release, releaseId) {
const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || []; const siteTags = release.site?.tags?.filter(tag => tag.inherit === true).map(tag => tag.id) || [];
const rawReleaseTags = release.tags.filter(Boolean) || []; const rawReleaseTags = release.tags || [];
const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string') const releaseTags = rawReleaseTags.some(tag => typeof tag === 'string')
? await matchTags(release.tags) // scraper returned raw tags ? await matchTags(release.tags) // scraper returned raw tags
: rawReleaseTags; // tags already matched by (outdated) scraper : rawReleaseTags; // tags already matched by (outdated) scraper