Added Fame Digital. Added actor release scraping to DDF Network. Improved q and Gamma scraper.
This commit is contained in:
@@ -1,101 +1,81 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
|
||||
const { d, ex, exa, get } = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
function scrapeLatest(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElements = $('.card.m-1').toArray();
|
||||
function scrapeAll(html, site, origin) {
|
||||
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
|
||||
const release = {};
|
||||
|
||||
return sceneElements.map((element) => {
|
||||
const sceneLinkElement = $(element).find('a').first();
|
||||
const title = sceneLinkElement.attr('title');
|
||||
const url = `${site.url}${sceneLinkElement.attr('href')}`;
|
||||
const entryId = url.split('/').slice(-1)[0];
|
||||
release.title = q('a', 'title');
|
||||
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
|
||||
[release.entryId] = release.url.split('/').slice(-1);
|
||||
|
||||
const date = moment.utc($(element).find('small[datetime]').attr('datetime'), 'YYYY-MM-DD HH:mm:ss').toDate();
|
||||
const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim())
|
||||
.toArray()
|
||||
.filter(actor => actor);
|
||||
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
|
||||
release.actors = qa('.card-subtitle a', true).filter(Boolean);
|
||||
|
||||
const duration = parseInt($(element).find('.card-info div:nth-child(2) .card-text').text(), 10) * 60;
|
||||
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
|
||||
if (duration) release.duration = duration;
|
||||
|
||||
const poster = sceneLinkElement.find('img').attr('data-src');
|
||||
release.poster = q('img').dataset.src;
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
duration,
|
||||
poster,
|
||||
rating: null,
|
||||
site,
|
||||
};
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
async function scrapeScene(html, url, _site) {
|
||||
const { q, qa, qd, qm, qp, qus } = ex(html);
|
||||
const release = {};
|
||||
|
||||
const entryId = url.split('/').slice(-1)[0];
|
||||
const title = $('meta[itemprop="name"]').attr('content');
|
||||
const description = $('.descr-box p').text(); // meta tags don't contain full description
|
||||
[release.entryId] = url.split('/').slice(-1);
|
||||
|
||||
const dateProp = $('meta[itemprop="uploadDate"]').attr('content');
|
||||
const date = dateProp
|
||||
? moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate()
|
||||
: moment.utc($('.title-border:nth-child(2) p').text(), 'MM.DD.YYYY').toDate();
|
||||
const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
|
||||
release.title = qm('itemprop=name');
|
||||
release.description = q('.descr-box p', true);
|
||||
release.date = qd('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|
||||
|| qd('.title-border:nth-child(2) p', 'MM.DD.YYYY');
|
||||
|
||||
const likes = Number($('.info-panel.likes .likes').text());
|
||||
const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
|
||||
release.actors = qa('.pornstar-card > a', 'title');
|
||||
release.tags = qa('.tags-tab .tags a', true);
|
||||
|
||||
const tags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
release.duration = parseInt(q('.icon-video-red + span', true), 10) * 60;
|
||||
release.likes = Number(q('.icon-like-red + span', true));
|
||||
|
||||
const poster = $('#video').attr('poster');
|
||||
const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
|
||||
release.poster = qp();
|
||||
release.photos = qus('.photo-slider-guest .card a');
|
||||
|
||||
const trailer540 = $('source[res="540"]').attr('src');
|
||||
const trailer720 = $('source[res="720"]').attr('src');
|
||||
release.trailer = qa('source[type="video/mp4"]').map(trailer => ({
|
||||
src: trailer.src,
|
||||
quality: Number(trailer.attributes.res.value),
|
||||
}));
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
description,
|
||||
actors,
|
||||
date,
|
||||
duration,
|
||||
tags,
|
||||
poster,
|
||||
photos,
|
||||
trailer: [
|
||||
{
|
||||
src: trailer720,
|
||||
quality: 720,
|
||||
},
|
||||
{
|
||||
src: trailer540,
|
||||
quality: 540,
|
||||
},
|
||||
],
|
||||
rating: {
|
||||
likes,
|
||||
},
|
||||
site,
|
||||
};
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchActorReleases(urls) {
|
||||
// DDF Network and DDF Network Stream list all scenes, exclude
|
||||
const sources = urls.filter(url => !/ddfnetwork/.test(url));
|
||||
|
||||
const releases = await Promise.all(sources.map(async (url) => {
|
||||
const { html } = await get(url);
|
||||
|
||||
return scrapeAll(html, null, new URL(url).origin);
|
||||
}));
|
||||
|
||||
// DDF cross-releases scenes between sites, filter duplicates by entryId
|
||||
return Object.values(releases
|
||||
.flat()
|
||||
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) // sort by date so earliest scene remains
|
||||
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
|
||||
}
|
||||
|
||||
async function scrapeProfile(html, _url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const { q, qa, qus } = ex(html);
|
||||
|
||||
const keys = Array.from(document.querySelectorAll('.about-title'), el => el.textContent.trim().replace(':', ''));
|
||||
const values = Array.from(document.querySelectorAll('.about-info'), (el) => {
|
||||
const keys = qa('.about-title', true).map(key => slugify(key, { delimiter: '_' }));
|
||||
const values = qa('.about-info').map((el) => {
|
||||
if (el.children.length > 0) {
|
||||
return Array.from(el.children, child => child.textContent.trim()).join(', ');
|
||||
}
|
||||
@@ -104,9 +84,7 @@ async function scrapeProfile(html, _url, actorName) {
|
||||
});
|
||||
|
||||
const bio = keys.reduce((acc, key, index) => {
|
||||
if (values[index] === '-') {
|
||||
return acc;
|
||||
}
|
||||
if (values[index] === '-') return acc;
|
||||
|
||||
return {
|
||||
...acc,
|
||||
@@ -114,45 +92,49 @@ async function scrapeProfile(html, _url, actorName) {
|
||||
};
|
||||
}, {});
|
||||
|
||||
const descriptionEl = document.querySelector('.description-box');
|
||||
const avatarEl = document.querySelector('.pornstar-details .card-img-top');
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
profile.birthdate = moment.utc(bio.Birthday, 'MMMM DD, YYYY').toDate();
|
||||
if (bio.Nationality) profile.nationality = bio.Nationality;
|
||||
profile.description = q('.description-box', true);
|
||||
profile.birthdate = d(bio.birthday, 'MMMM DD, YYYY');
|
||||
|
||||
if (bio['Bra size']) [profile.bust] = bio['Bra size'].match(/\d+\w+/);
|
||||
if (bio.Waist) profile.waist = Number(bio.Waist.match(/\d+/)[0]);
|
||||
if (bio.Hips) profile.hip = Number(bio.Hips.match(/\d+/)[0]);
|
||||
if (bio.nationality) profile.nationality = bio.nationality;
|
||||
|
||||
if (bio.Height) profile.height = Number(bio.Height.match(/\d{2,}/)[0]);
|
||||
if (bio.bra_size) [profile.bust] = bio.bra_size.match(/\d+\w+/);
|
||||
if (bio.waist) profile.waist = Number(bio.waist.match(/\d+/)[0]);
|
||||
if (bio.hips) profile.hip = Number(bio.hips.match(/\d+/)[0]);
|
||||
|
||||
if (bio['Tit Style'] && bio['Tit Style'].match('Enhanced')) profile.naturalBoobs = false;
|
||||
if (bio['Tit Style'] && bio['Tit Style'].match('Natural')) profile.naturalBoobs = true;
|
||||
if (bio.height) profile.height = Number(bio.height.match(/\d{2,}/)[0]);
|
||||
|
||||
if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true;
|
||||
if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true;
|
||||
if (bio.tit_style && /Enhanced/.test(bio.tit_style)) profile.naturalBoobs = false;
|
||||
if (bio.tit_style && /Natural/.test(bio.tit_style)) profile.naturalBoobs = true;
|
||||
|
||||
if (bio['Hair Style']) profile.hair = bio['Hair Style'].split(',')[0].trim().toLowerCase();
|
||||
if (bio['Eye Color']) profile.eyes = bio['Eye Color'].match(/\w+/)[0].toLowerCase();
|
||||
if (bio.body_art && /Tattoo/.test(bio.body_art)) profile.hasTattoos = true;
|
||||
if (bio.body_art && /Piercing/.test(bio.body_art)) profile.hasPiercings = true;
|
||||
|
||||
if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]);
|
||||
if (bio.hair_style) profile.hair = bio.hair_style.split(',')[0].trim().toLowerCase();
|
||||
if (bio.eye_color) profile.eyes = bio.eye_color.match(/\w+/)[0].toLowerCase();
|
||||
|
||||
if (descriptionEl) profile.description = descriptionEl.textContent.trim();
|
||||
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
|
||||
|
||||
const avatarEl = q('.pornstar-details .card-img-top');
|
||||
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
|
||||
|
||||
profile.releases = await fetchActorReleases(qus('.find-me-tab li a'));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
|
||||
const url = site.parameters?.native
|
||||
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
|
||||
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
|
||||
|
||||
console.log(url);
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
return scrapeAll(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
|
||||
Reference in New Issue
Block a user