DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added proper Gamma parameters to 21Sextury sites.

This commit is contained in:
ThePendulum 2020-02-13 23:05:28 +01:00
parent 14b8993d10
commit 5e461f1f1a
4 changed files with 128 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
seeds/02_sites.js
View File

@ -154,6 +154,9 @@ const sites = [
url: 'https://www.analteenangels.com',
description: 'AnalTeenAngels is presented by the 21Sextury nextwork and features young, European teens in hardcore anal porn. Watch these barely legal teens have their first anal sex and give up their ass for some anal pounding!',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
},
},
{
slug: 'assholefever',
@ -167,16 +170,31 @@ const sites = [
slug: 'buttplays',
name: 'Butt Plays',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'clubsandy',
name: 'Club Sandy',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'deepthroatfrenzy',
name: 'Deepthroat Frenzy',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'dpfanatics',
@ -184,6 +202,9 @@ const sites = [
url: 'https://www.dpfanatics.com',
description: 'Welcome to DPFanatics, brought to you by 21Sextury. DP Fanatics brings you the best DP sex and double penetration porn you can find. Double vaginal penetration, double anal, amateur and teen DP inside!',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
},
},
{
slug: 'footsiebabes',
@ -191,11 +212,21 @@ const sites = [
url: 'https://www.footsiebabes.com',
description: 'Welcome to FootsieBabes.com, bringing you the best foot porn, teen feet and foot worship you can find on the net. Watch stocking porn, footjobs, feet tickling and more inside!',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'gapeland',
name: 'Gapeland',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'lezcuties',
@ -203,66 +234,129 @@ const sites = [
url: 'https://www.lezcuties.com',
description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
},
},
{
slug: 'onlyswallows',
name: 'Only Swallows',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'alettaoceanempire',
name: 'Aletta Ocean Empire',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'analqueenalysa',
name: 'Anal Queen Alysa',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'blueangellive',
name: 'Blue Angel Live',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'pixandvideo',
name: 'Pix and Video',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'cheatingwhorewives',
name: 'Cheating Whore Wives',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'cutiesgalore',
name: 'Cuties Galore',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'hotmilfclub',
name: 'Hot MILF Club',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'letsplaylez',
name: 'Lets Play Lez',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'nudefightclub',
name: 'Nude Fight Club',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'sexwithkathianobili',
name: 'Sex With Kathia Nobili',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
{
slug: 'sweetsophiemoone',
name: 'Sweet Sophie Moone',
network: '21sextury',
parameters: {
referer: 'https://www.21sextury.com',
scene: 'https://www.21sextury.com/en/video',
photos: 'https://www.21sextury.com/en/photo',
},
},
// ADULT TIME
{
@ -272,7 +366,7 @@ const sites = [
network: 'adulttime',
parameters: {
referer: 'https://freetour.adulttime.com/en/join',
deep: 'https://21sextury.com/en/video/',
deep: 'https://21sextury.com/en/video',
scene: false,
},
},
@ -283,7 +377,7 @@ const sites = [
network: 'adulttime',
parameters: {
referer: 'https://freetour.bubblegumdungeon.com/en/join',
deep: 'https://21sextury.com/en/video/',
deep: 'https://21sextury.com/en/video',
scene: false,
},
},
@ -1685,7 +1779,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
{
@ -1697,7 +1791,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
{
@ -1709,7 +1803,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
{
@ -1721,7 +1815,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
{
@ -1733,7 +1827,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
{
@ -1745,7 +1839,7 @@ const sites = [
latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
photos: 'https://www.fantasymassage.com/en/photo',
},
},
// GIRLSWAY
@ -1881,7 +1975,7 @@ const sites = [
referer: 'https://freetour.adulttime.com/en/blackforwife',
useGamma: true,
scene: false,
deep: 'https://21sextury.com/en/video/',
deep: 'https://21sextury.com/en/video',
},
},
// JULES JORDAN

8
src/scrape-releases.js
View File

@ -35,11 +35,11 @@ async function findSite(url, release) {
async function scrapeRelease(source, basicRelease = null, type = 'scene', preflight) {
// profile scraper may return either URLs or pre-scraped scenes
const sourceIsUrl = typeof source === 'string';
const url = sourceIsUrl ? source : source?.url;
const release = sourceIsUrl ? basicRelease : source;
const sourceIsUrlOrEmpty = typeof source === 'string' || source === undefined;
const url = sourceIsUrlOrEmpty ? source : source?.url;
const release = sourceIsUrlOrEmpty ? basicRelease : source;
const site = await findSite(url, release);
const site = basicRelease?.site || await findSite(url, release);
if (!site) {
throw new Error(`Could not find site ${url} in database`);

2
src/scrape-sites.js
View File

@ -98,7 +98,7 @@ async function deepFetchReleases(baseReleases, preflight) {
return Promise.map(baseReleases, async (release) => {
if (release.url || (release.path && release.site)) {
try {
const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene', preflight);
const fullRelease = await scrapeRelease(release.url, release, 'scene', preflight);
if (fullRelease) {
return {

29
src/scrapers/gamma.js
View File

@ -14,11 +14,15 @@ const slugify = require('../utils/slugify');
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
? `${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}/${albumPath.split('/').slice(-2).join('/')}`;
}
return `${site.url}${albumPath}`;
if (site.url) {
return `${site.url}${albumPath}`;
}
return null;
}
async function fetchPhotos(url) {
@ -59,6 +63,10 @@ function scrapePhotos(html) {
async function getPhotos(albumPath, site) {
const albumUrl = getAlbumUrl(albumPath, site);
if (!albumUrl) {
return [];
}
try {
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -70,7 +78,7 @@ async function getPhotos(albumPath, site) {
const otherPages = Array.from({ length: Number(lastPage) }, (_value, index) => index + 1).slice(1);
const otherPhotos = await Promise.map(otherPages, async (page) => {
const pageUrl = `${site.url}/${albumPath}/${page}`;
const pageUrl = `${albumUrl}/${page}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml, site);
@ -107,7 +115,7 @@ async function scrapeApiReleases(json, site) {
release.path = `/${scene.url_title}/${release.entryId}`;
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
@ -469,12 +477,15 @@ async function fetchScene(url, site, release) {
}
const deepUrl = getDeepUrl(url, site, release);
const res = await bhttp.get(deepUrl);
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
if (deepUrl) {
const res = await bhttp.get(deepUrl);
return { ...scene, deepUrl };
if (res.statusCode === 200) {
const scene = await scrapeScene(res.body.toString(), url, site, release);
return { ...scene, deepUrl };
}
}
return null;