boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Merged DDF Network with Porn World. Fixed and updated DDF/PW scraper.

Browse Source
This commit is contained in:
ThePendulum
2020-07-12 22:36:53 +02:00
parent 70c60e93ac
commit a7707b7b28
66 changed files with 94 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
src/scrapers/ddfnetwork.js
View File

@@ -2,53 +2,60 @@
const bhttp = require('bhttp');
const { ed, ex, exa, get } = require('../utils/q');
const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */
function scrapeAll(html, site, origin) {
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
function scrapeAll(scenes, site, origin) {
return scenes.map(({ query }) => {
const release = {};
release.title = q('a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1);
release.title = query.q('.card-title a, .videoContent h4 a', 'title');
release.url = `${site?.url || origin || 'https://pornworld.com'}${query.q('a', 'href')}`;
release.entryId = release.url.match(/\/(\d+)$/)[1];
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean);
release.date = query.date('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = query.all('.card-subtitle a, .featuring a', true).filter(Boolean);
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
release.description = query.q('h4 + p', true);
const duration = parseInt(query.q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration;
else release.duration = query.dur('.time');
release.poster = q('img').dataset.src;
release.poster = query.img();
return release;
});
}
async function scrapeScene(html, url, _site) {
const { qu } = ex(html);
async function scrapeScene({ query }, url, _site) {
const release = {};
[release.entryId] = url.split('/').slice(-1);
release.entryId = url.match(/\/(\d+)$/)[1];
release.title = qu.meta('itemprop=name');
release.description = qu.q('.descr-box p', true);
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
release.title = query.meta('itemprop=name') || query.q('.video-title h1', true) || query.q('.about-text .story-title') || query.q('h3', true);
release.description = query.q('.descr-box p', true) || query.q('.about-text p:not(.story-title)', true) || query.text('.description p');
release.actors = qu.all('.pornstar-card > a', 'title');
release.tags = qu.all('.tags-tab .tags a', true);
release.date = query.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| query.date('.actors time', 'MMMM DD, YYYY')
|| query.date('.title-border:nth-child(2) p', 'MM.DD.YYYY')
|| query.date('.length', 'MMMM DD, YYYY', /\w+ \d{2}, \d{4}/);
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
release.likes = Number(qu.q('.icon-like-red + span', true));
if (query.exists('.pornstar-card > a')) release.actors = query.all('.pornstar-card > a', 'title');
else if (query.exists('.actors a')) release.actors = query.all('.actors a', true);
release.poster = qu.poster();
release.photos = qu.urls('.photo-slider-guest .card a');
if (query.exists('.tags-tab')) release.tags = query.all('.tags-tab .tags a', true);
else if (query.exists('.tags-box')) release.tags = query.all('.tags-box .tags li', true);
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
release.duration = parseInt(query.q('.icon-video-red + span', true), 10) * 60 || query.dur('.length') || null;
release.likes = Number(query.q('.icon-like-red + span', true)) || null;
release.poster = query.poster() || query.poster('dl8-video');
release.photos = query.urls('.photo-slider-guest .card a');
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src,
quality: Number(trailer.attributes.res.value),
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
}));
return release;
@@ -59,9 +66,9 @@ async function fetchActorReleases(urls) {
const sources = urls.filter(url => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url);
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
return scrapeAll(html, null, new URL(url).origin);
return res.ok ? scrapeAll(res.items, null, new URL(url).origin) : null;
}));
// DDF cross-releases scenes between sites, filter duplicates by entryId
@@ -71,11 +78,9 @@ async function fetchActorReleases(urls) {
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
}
async function scrapeProfile(html, _url, actorName) {
const { qu } = ex(html);
const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
const values = qu.all('.about-info').map((el) => {
async function scrapeProfile({ query }, _url, actorName) {
const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
const values = query.all('.about-info').map((el) => {
if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', ');
}
@@ -96,8 +101,8 @@ async function scrapeProfile(html, _url, actorName) {
name: actorName,
};
profile.description = qu.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
profile.description = query.q('.description-box', true);
profile.birthdate = qu.extractDate(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality;
@@ -118,34 +123,37 @@ async function scrapeProfile(html, _url, actorName) {
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = qu.q('.pornstar-details .card-img-top');
const avatarEl = query.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
profile.releases = await fetchActorReleases(query.urls('.find-me-tab li a'));
return profile;
}
async function fetchLatest(site, page = 1) {
async function fetchLatest(channel, page = 1) {
/* ddfnetwork.com redirects to pornworld.com
const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
*/
const res = await bhttp.get(url);
const url = channel.parameters?.latest || `${channel.url}/videos/search/latest/ever/allsite/-/${page}`;
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card), .allVideos .videoBlock');
if (res.statusCode === 200) {
return scrapeAll(res.body.toString(), site);
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.statusCode;
return res.status;
}
async function fetchScene(url, site) {
// DDF's main site moved to Porn World
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
const res = await bhttp.get(url);
const res = await qu.get(url, '.content, #content, .taspVideoPage');
return scrapeScene(res.body.toString(), url, site);
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchProfile(actorName) {
@@ -155,7 +163,7 @@ async function fetchProfile(actorName) {
word: actorName,
},
{
decodeJSON: true,
decodeJSON: false,
headers: {
'x-requested-with': 'XMLHttpRequest',
},

4
src/scrapers/scrapers.js
View File

@@ -83,7 +83,7 @@ module.exports = {
brazzers,
burningangel,
cherrypimps,
ddfnetwork,
pornworld: ddfnetwork,
digitalplayground,
dogfart,
dogfartnetwork: dogfart,
@@ -155,7 +155,7 @@ module.exports = {
brazzers,
burningangel,
cherrypimps,
ddfnetwork,
pornworld: ddfnetwork,
deeper: vixen,
deeplush: nubiles,
devilsfilm: famedigital,