boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added Fame Digital. Added actor release scraping to DDF Network. Improved q and Gamma scraper.

Browse Source
This commit is contained in:
ThePendulum
2020-02-06 23:15:28 +01:00
parent db14eaa5f9
commit 6e1de52a40
42 changed files with 752 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
src/scrapers/ddfnetwork.js
View File

@@ -1,101 +1,81 @@
'use strict';
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const { d, ex, exa, get } = require('../utils/q');
const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.card.m-1').toArray();
function scrapeAll(html, site, origin) {
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
const release = {};
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('a').first();
const title = sceneLinkElement.attr('title');
const url = `${site.url}${sceneLinkElement.attr('href')}`;
const entryId = url.split('/').slice(-1)[0];
release.title = q('a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1);
const date = moment.utc($(element).find('small[datetime]').attr('datetime'), 'YYYY-MM-DD HH:mm:ss').toDate();
const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim())
.toArray()
.filter(actor => actor);
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean);
const duration = parseInt($(element).find('.card-info div:nth-child(2) .card-text').text(), 10) * 60;
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration;
const poster = sceneLinkElement.find('img').attr('data-src');
release.poster = q('img').dataset.src;
return {
url,
entryId,
title,
actors,
date,
duration,
poster,
rating: null,
site,
};
return release;
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
async function scrapeScene(html, url, _site) {
const { q, qa, qd, qm, qp, qus } = ex(html);
const release = {};
const entryId = url.split('/').slice(-1)[0];
const title = $('meta[itemprop="name"]').attr('content');
const description = $('.descr-box p').text(); // meta tags don't contain full description
[release.entryId] = url.split('/').slice(-1);
const dateProp = $('meta[itemprop="uploadDate"]').attr('content');
const date = dateProp
? moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate()
: moment.utc($('.title-border:nth-child(2) p').text(), 'MM.DD.YYYY').toDate();
const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
release.title = qm('itemprop=name');
release.description = q('.descr-box p', true);
release.date = qd('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qd('.title-border:nth-child(2) p', 'MM.DD.YYYY');
const likes = Number($('.info-panel.likes .likes').text());
const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
release.actors = qa('.pornstar-card > a', 'title');
release.tags = qa('.tags-tab .tags a', true);
const tags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.duration = parseInt(q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(q('.icon-like-red + span', true));
const poster = $('#video').attr('poster');
const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
release.poster = qp();
release.photos = qus('.photo-slider-guest .card a');
const trailer540 = $('source[res="540"]').attr('src');
const trailer720 = $('source[res="720"]').attr('src');
release.trailer = qa('source[type="video/mp4"]').map(trailer => ({
src: trailer.src,
quality: Number(trailer.attributes.res.value),
}));
return {
url,
entryId,
title,
description,
actors,
date,
duration,
tags,
poster,
photos,
trailer: [
{
src: trailer720,
quality: 720,
},
{
src: trailer540,
quality: 540,
},
],
rating: {
likes,
},
site,
};
return release;
}
async function fetchActorReleases(urls) {
// DDF Network and DDF Network Stream list all scenes, exclude
const sources = urls.filter(url => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url);
return scrapeAll(html, null, new URL(url).origin);
}));
// DDF cross-releases scenes between sites, filter duplicates by entryId
return Object.values(releases
.flat()
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
}
async function scrapeProfile(html, _url, actorName) {
const { document } = new JSDOM(html).window;
const { q, qa, qus } = ex(html);
const keys = Array.from(document.querySelectorAll('.about-title'), el => el.textContent.trim().replace(':', ''));
const values = Array.from(document.querySelectorAll('.about-info'), (el) => {
const keys = qa('.about-title', true).map(key => slugify(key, { delimiter: '_' }));
const values = qa('.about-info').map((el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
}
@@ -104,9 +84,7 @@ async function scrapeProfile(html, _url, actorName) {
});
const bio = keys.reduce((acc, key, index) => {
if (values[index] === '-') {
return acc;
}
if (values[index] === '-') return acc;
return {
...acc,
@@ -114,45 +92,49 @@ async function scrapeProfile(html, _url, actorName) {
};
}, {});
const descriptionEl = document.querySelector('.description-box');
const avatarEl = document.querySelector('.pornstar-details .card-img-top');
const profile = {
name: actorName,
};
profile.birthdate = moment.utc(bio.Birthday, 'MMMM DD, YYYY').toDate();
if (bio.Nationality) profile.nationality = bio.Nationality;
profile.description = q('.description-box', true);
profile.birthdate = d(bio.birthday, 'MMMM DD, YYYY');
if (bio['Bra size']) [profile.bust] = bio['Bra size'].match(/\d+\w+/);
if (bio.Waist) profile.waist = Number(bio.Waist.match(/\d+/)[0]);
if (bio.Hips) profile.hip = Number(bio.Hips.match(/\d+/)[0]);
if (bio.nationality) profile.nationality = bio.nationality;
if (bio.Height) profile.height = Number(bio.Height.match(/\d{2,}/)[0]);
if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/);
if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]);
if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]);
if (bio['Tit Style'] && bio['Tit Style'].match('Enhanced')) profile.naturalBoobs = false;
if (bio['Tit Style'] && bio['Tit Style'].match('Natural')) profile.naturalBoobs = true;
if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]);
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false;
if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true;
if (bio['Hair Style']) profile.hair = bio['Hair Style'].split(',')[0].trim().toLowerCase();
if (bio['Eye Color']) profile.eyes = bio['Eye Color'].match(/\w+/)[0].toLowerCase();
if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true;
if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true;
if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]);
if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase();
if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase();
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qus('.find-me-tab li a'));
return profile;
}
async function fetchLatest(site, page = 1) {
const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
console.log(url);
const res = await bhttp.get(url);
return scrapeLatest(res.body.toString(), site);
return scrapeAll(res.body.toString(), site);
}
async function fetchScene(url, site) {

72
src/scrapers/famedigital.js Normal file
View File

@@ -0,0 +1,72 @@
'use strict';
const { fetchLatest, fetchApiLatest, fetchUpcoming, fetchApiUpcoming, fetchScene, fetchProfile, fetchApiProfile } = require('./gamma');
function extractLowArtActors(release) {
const actors = release.title
.replace(/solo/i, '')
.split(/,|\band\b/ig)
.map(actor => actor.trim());
return {
...release,
actors,
};
}
async function networkFetchLatest(site, page = 1) {
if (site.parameters?.api) return fetchApiLatest(site, page, false);
const releases = await fetchLatest(site, page);
if (site.slug === 'lowartfilms') {
return releases.map(release => extractLowArtActors(release));
}
return releases;
}
async function networkFetchScene(url, site) {
const release = await fetchScene(url, site);
if (site.slug === 'lowartfilms') {
return extractLowArtActors(release);
}
return release;
}
async function networkFetchUpcoming(site, page = 1) {
if (site.parameters?.api) return fetchApiUpcoming(site, page, true);
return fetchUpcoming(site, page);
}
async function networkFetchProfile(actorName) {
// not all Fame Digital sites offer Gamma actors
const [devils, rocco, peter] = await Promise.all([
fetchApiProfile(actorName, 'devilsfilm', true),
fetchApiProfile(actorName, 'roccosiffredi'),
fetchProfile(actorName, 'peternorth', true),
]);
if (devils || rocco || peter) {
const releases = [].concat(devils?.releases || [], rocco?.releases || [], peter?.releases || []);
return {
...peter,
...rocco,
...devils,
releases,
};
}
return null;
}
module.exports = {
fetchLatest: networkFetchLatest,
fetchProfile: networkFetchProfile,
fetchScene: networkFetchScene,
fetchUpcoming: networkFetchUpcoming,
};

106
src/scrapers/gamma.js
View File

@@ -111,18 +111,23 @@ function scrapeAll(html, site, useNetworkUrl) {
const scenesElements = $('li[data-itemtype=scene]').toArray();
return scenesElements.map((element) => {
const release = {};
const sceneLinkElement = $(element).find('.sceneTitle a');
const url = `${useNetworkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
release.url = `${useNetworkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`;
release.title = sceneLinkElement.attr('title');
const entryId = $(element).attr('data-itemid');
release.entryId = $(element).attr('data-itemid');
const date = moment
.utc($(element).find('.sceneDate').text(), 'MM-DD-YYYY')
.toDate();
const dateEl = $(element).find('.sceneDate').text() || null;
if (dateEl) {
release.date = moment
.utc($(element).find('.sceneDate').text(), ['MM-DD-YYYY', 'YYYY-MM-DD'])
.toDate();
}
const actors = $(element).find('.sceneActors a')
release.actors = $(element).find('.sceneActors a')
.map((actorIndex, actorElement) => $(actorElement).attr('title'))
.toArray();
@@ -130,27 +135,17 @@ function scrapeAll(html, site, useNetworkUrl) {
.toArray()
.map(value => Number($(value).text()));
const poster = $(element).find('.imgLink img').attr('data-original');
const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
release.rating = { likes, dislikes };
return {
url,
entryId,
title,
actors,
director: 'Mason',
date,
poster,
trailer: {
src: trailer,
quality: 224,
},
rating: {
likes,
dislikes,
},
site,
const posterEl = $(element).find('.imgLink img');
if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src');
release.teaser = {
src: `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`,
quality: 224,
};
return release;
});
}
@@ -161,33 +156,41 @@ async function scrapeScene(html, url, site) {
const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
const [data, data2] = JSON.parse(json);
const [data, data2] = json ? JSON.parse(json) : [];
const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1));
[release.entryId] = new URL(url).pathname.split('/').slice(-1);
release.title = data?.name || videoData.playerOptions.sceneInfos.sceneTitle;
release.title = data.name;
release.description = data.description;
// date in data object is not the release date of the scene, but the date the entry was added
// date in data object is not the release date of the scene, but the date the entry was added; only use as fallback
const dateString = $('.updatedDate').first().text().trim();
const dateMatch = dateString.match(/\d{2,4}-\d{2}-\d{2,4}/)?.[0];
release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
release.director = data.director?.[0].name || data2?.director?.[0].name;
release.actors = (data.actor || data2.actor).map(actor => actor.name);
const hasTrans = (data.actor || data2.actor).some(actor => actor.gender === 'shemale');
if (dateMatch) release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate();
else if (data?.dateCreated) release.date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate();
else release.date = videoData.playerOptions.sceneInfos.sceneReleaseDate;
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars };
if (data) {
release.description = data.description;
release.director = data.director?.[0].name || data2?.director?.[0].name;
release.duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
const actors = data?.actor || data2?.actor || [];
const hasTrans = actors.some(actor => actor.gender === 'shemale');
release.actors = actors.map(actor => actor.name);
const rawTags = data.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars };
release.duration = moment.duration(data.duration.slice(2)).asSeconds();
const rawTags = data.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
}
release.poster = videoData.picPreview;
release.photos = await getPhotos($('.picturesItem a').attr('href'), site);
const photoLink = $('.picturesItem a').attr('href');
if (photoLink) release.photos = await getPhotos(photoLink, site);
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
release.trailer = [
@@ -297,6 +300,11 @@ async function fetchApiCredentials(referer) {
const body = res.body.toString();
const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
if (!apiLine) {
throw new Error(`Can not use Gamma API for ${referer}`);
}
const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
const apiData = JSON.parse(apiSerial);
@@ -321,7 +329,7 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`,
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug}`,
},
],
}, {
@@ -331,7 +339,11 @@ async function fetchApiLatest(site, page = 1, upcoming = false) {
encodeJSON: true,
});
return scrapeApiReleases(res.body.results[0].hits, site);
if (res.statuscode === 200 && res.body.results?.[0]?.hits) {
return scrapeApiReleases(res.body.results[0].hits, site);
}
return [];
}
async function fetchApiUpcoming(site) {
@@ -339,14 +351,14 @@ async function fetchApiUpcoming(site) {
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}/en/videos/AllCategories/0/${page}`;
const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`;
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`);
const res = await bhttp.get(`${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`);
return scrapeAll(res.body.toString(), site);
}
@@ -362,7 +374,7 @@ async function fetchActorScenes(actorName, apiUrl, siteSlug) {
requests: [
{
indexName: 'all_scenes',
params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
params: `query=&filters=sitename:${siteSlug}&hitsPerPage=36&maxValuesPerFacet=100&page=0&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["actors.name:${actorName}"]]`,
},
],
}, {
@@ -408,7 +420,7 @@ async function fetchProfile(actorName, siteSlug, altSearchUrl) {
async function fetchApiProfile(actorName, siteSlug) {
const actorSlug = encodeURI(actorName);
const referer = `https://www.${siteSlug}.com/en/search?query=${actorSlug}&tab=actors`;
const referer = `https://www.${siteSlug}.com/en/search`;
const { apiUrl } = await fetchApiCredentials(referer);

2
src/scrapers/score.js
View File

@@ -80,7 +80,7 @@ async function scrapeScene(html, url, site) {
const actorEl = qa('.stat').find(stat => /Featuring/.test(stat.textContent));
const actorString = qtext(actorEl);
release.actors = actorString?.split(/, and |, /g) || [];
release.actors = actorString?.split(/,\band\b|,/g).map(actor => actor.trim()) || [];
}
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;

3
src/scrapers/scrapers.js
View File

@@ -26,6 +26,7 @@ const bangbros = require('./bangbros');
const blowpass = require('./blowpass');
const brazzers = require('./brazzers');
const ddfnetwork = require('./ddfnetwork');
const famedigital = require('./famedigital');
const evilangel = require('./evilangel');
const julesjordan = require('./julesjordan');
const kellymadison = require('./kellymadison');
@@ -60,6 +61,7 @@ module.exports = {
digitalplayground,
dogfart,
dogfartnetwork: dogfart,
famedigital,
evilangel,
fakehub,
jayrock,
@@ -96,6 +98,7 @@ module.exports = {
brazzers,
ddfnetwork,
digitalplayground,
famedigital,
evilangel,
fakehub,
freeones,

1
src/scrapers/xempire.js
View File

@@ -14,6 +14,7 @@ async function fetchScene(url, site) {
// const siteUrl = siteDomain && `https://www.${siteDomain}`;
release.channel = siteSlug;
release.director = 'Mason';
return release;
}