forked from DebaucheryLibrarian/traxxx
Added Nubiles network.
This commit is contained in:
149
src/scrapers/nubiles.js
Normal file
149
src/scrapers/nubiles.js
Normal file
@@ -0,0 +1,149 @@
|
||||
'use strict';
|
||||
|
||||
const { get, geta, ctxa } = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { heightToCm } = require('../utils/convert');
|
||||
|
||||
const slugUrlMap = {
|
||||
nubiles: 'https://www.nubiles.net',
|
||||
nubilesporn: 'https://www.nubiles-porn.com',
|
||||
};
|
||||
|
||||
async function getPhotos(albumUrl) {
|
||||
const thumbnails = await geta(albumUrl, '.photo-thumb');
|
||||
|
||||
return thumbnails
|
||||
? thumbnails.map(({ q }) => q('source').srcset)
|
||||
: [];
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, site, origin) {
|
||||
return scenes.map(({ q, qa, qu, qd }) => {
|
||||
const release = {};
|
||||
|
||||
release.title = q('.title a', true);
|
||||
|
||||
const url = qu('.title a').split('?')[0];
|
||||
const channelUrl = qu('.site-link');
|
||||
|
||||
if (/^http/.test(url)) {
|
||||
const { pathname } = new URL(url);
|
||||
release.entryId = pathname.split('/')[3];
|
||||
|
||||
if (channelUrl) release.url = `${channelUrl}${pathname}`;
|
||||
else release.url = url;
|
||||
} else {
|
||||
release.entryId = url.split('/')[3];
|
||||
|
||||
if (channelUrl) release.url = `${channelUrl}${url}`;
|
||||
else if (site?.url) release.url = `${site.url}${url}`;
|
||||
else if (origin) release.url = `${origin}${url}`;
|
||||
}
|
||||
|
||||
release.date = qd('.date', 'MMM D, YYYY');
|
||||
release.actors = qa('.models a.model', true);
|
||||
|
||||
const poster = q('img').dataset.original;
|
||||
release.poster = [
|
||||
poster.replace('_640', '_1280'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.stars = Number(q('.rating', true));
|
||||
release.likes = Number(q('.likes', true));
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ q, qa, qd, qp, qu, qi }, url, site) {
|
||||
const release = {};
|
||||
|
||||
const { origin, pathname } = new URL(url);
|
||||
release.url = `${origin}${pathname}`;
|
||||
|
||||
release.entryId = new URL(url).pathname.split('/')[3];
|
||||
release.title = q('.content-pane-title h2', true);
|
||||
release.description = q('.content-pane-column div', true);
|
||||
|
||||
release.date = qd('.date', 'MMM D, YYYY');
|
||||
|
||||
release.actors = qa('.content-pane-performers .model', true);
|
||||
release.tags = qa('.categories a', true);
|
||||
|
||||
release.poster = qp() || qi('.fake-video-player img');
|
||||
release.trailer = qa('source').map(source => ({
|
||||
src: source.src,
|
||||
quality: Number(source.getAttribute('res')),
|
||||
}));
|
||||
|
||||
release.stars = Number(q('.score', true));
|
||||
release.likes = Number(q('#likecount', true));
|
||||
|
||||
const albumLink = qu('.content-pane-related-links a[href*="gallery"]');
|
||||
if (albumLink) release.photos = await getPhotos(`${site.url}${albumLink}`);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ q, qa, qi, qu }, _actorName, origin) {
|
||||
const profile = {};
|
||||
|
||||
const keys = qa('.model-profile h5', true);
|
||||
const values = qa('.model-profile h5 + p', true);
|
||||
|
||||
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, { delimiter: '_' })]: values[index] }), {});
|
||||
|
||||
profile.age = Number(bio.age);
|
||||
profile.description = q('.model-bio', true);
|
||||
|
||||
profile.residencePlace = bio.location;
|
||||
|
||||
profile.height = heightToCm(bio.height);
|
||||
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map(v => Number(v) || v);
|
||||
|
||||
profile.avatar = qi('.model-profile img');
|
||||
|
||||
const releases = qa('.content-grid-item').filter(el => /video\//.test(qu(el, '.img-wrapper a'))); // filter out photos
|
||||
profile.releases = scrapeAll(ctxa(releases), null, origin);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/video/gallery/${(page - 1) * 12}`;
|
||||
const qLatest = await geta(url, '.content-grid-item');
|
||||
|
||||
return qLatest && scrapeAll(qLatest, site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const qScene = await get(url);
|
||||
|
||||
return qScene && scrapeScene(qScene, url, site);
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName, siteSlug) {
|
||||
const firstLetter = actorName.charAt(0).toLowerCase();
|
||||
const origin = slugUrlMap[siteSlug] || `https://www.${siteSlug}.com`;
|
||||
|
||||
const url = `${origin}/model/alpha/${firstLetter}`;
|
||||
const { qa } = await get(url);
|
||||
|
||||
const modelPath = qa('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName));
|
||||
|
||||
if (modelPath) {
|
||||
const modelUrl = `${origin}${modelPath}`;
|
||||
const qModel = await get(modelUrl);
|
||||
|
||||
if (qModel) return scrapeProfile(qModel, actorName, origin);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
};
|
||||
@@ -33,6 +33,7 @@ const mindgeek = require('./mindgeek');
|
||||
const mofos = require('./mofos');
|
||||
const naturals = require('./21naturals');
|
||||
const naughtyamerica = require('./naughtyamerica');
|
||||
const nubiles = require('./nubiles');
|
||||
const perfectgonzo = require('./perfectgonzo');
|
||||
const pervcity = require('./pervcity');
|
||||
const pornhub = require('./pornhub');
|
||||
@@ -65,13 +66,12 @@ module.exports = {
|
||||
blowpass,
|
||||
brazzers,
|
||||
ddfnetwork,
|
||||
metrohd,
|
||||
digitalplayground,
|
||||
dogfart,
|
||||
dogfartnetwork: dogfart,
|
||||
famedigital,
|
||||
evilangel,
|
||||
fakehub,
|
||||
famedigital,
|
||||
fantasymassage,
|
||||
girlsway,
|
||||
insex,
|
||||
@@ -81,16 +81,18 @@ module.exports = {
|
||||
kink,
|
||||
legalporno,
|
||||
men,
|
||||
metrohd,
|
||||
mikeadriano,
|
||||
milehighmedia,
|
||||
mindgeek,
|
||||
mofos,
|
||||
naughtyamerica,
|
||||
nubiles,
|
||||
perfectgonzo,
|
||||
pervcity,
|
||||
pornpros,
|
||||
private: privateNetwork,
|
||||
puretaboo,
|
||||
naughtyamerica,
|
||||
realitykings,
|
||||
score,
|
||||
teamskeet,
|
||||
@@ -102,20 +104,23 @@ module.exports = {
|
||||
xempire,
|
||||
},
|
||||
actors: {
|
||||
// ordered by data priority
|
||||
'21sextury': sextury,
|
||||
anilos: nubiles,
|
||||
babes,
|
||||
bangbros,
|
||||
blowpass,
|
||||
boobpedia,
|
||||
brattysis: nubiles,
|
||||
brazzers,
|
||||
ddfnetwork,
|
||||
deeplush: nubiles,
|
||||
digitalplayground,
|
||||
famedigital,
|
||||
evilangel,
|
||||
fakehub,
|
||||
famedigital,
|
||||
freeones,
|
||||
freeonesLegacy,
|
||||
hotcrazymess: nubiles,
|
||||
iconmale,
|
||||
julesjordan,
|
||||
kellymadison,
|
||||
@@ -125,9 +130,14 @@ module.exports = {
|
||||
milehighmedia,
|
||||
mofos,
|
||||
naughtyamerica,
|
||||
nfbusty: nubiles,
|
||||
nubilefilms: nubiles,
|
||||
nubiles,
|
||||
nubilesporn: nubiles,
|
||||
pornhub,
|
||||
realitykings,
|
||||
score,
|
||||
thatsitcomshow: nubiles,
|
||||
transangels,
|
||||
twistys,
|
||||
wicked,
|
||||
|
||||
@@ -1,105 +1,50 @@
|
||||
'use strict';
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
const bhttp = require('bhttp');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
const { get, geta } = require('../utils/q');
|
||||
|
||||
const { matchTags } = require('../tags');
|
||||
function scrapeLatest(scenes, site) {
|
||||
return scenes.map(({ q, qa, qu, qd }) => {
|
||||
const release = {};
|
||||
|
||||
function scrapeLatest(html, site) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const sceneElements = $('.scenes-latest').toArray();
|
||||
release.title = q('.title a', true);
|
||||
|
||||
return sceneElements.map((element) => {
|
||||
const actors = $(element).find('.actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
const pathname = qu('.title a');
|
||||
release.entryId = pathname.split('/')[3];
|
||||
release.url = `${site.url}${pathname}`;
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
stars,
|
||||
},
|
||||
site,
|
||||
};
|
||||
release.date = qd('.date', 'MMM DD, YYYY');
|
||||
release.actors = qa('.models a.model', true);
|
||||
|
||||
release.poster = q('img').dataset.original;
|
||||
|
||||
release.stars = Number(q('.rating', true));
|
||||
release.likes = Number(q('.likes', true));
|
||||
|
||||
console.log(release);
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeUpcoming(html, site) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
const sceneElements = $('.scenes-upcoming').toArray();
|
||||
function scrapeScene(({ q }), _site) {
|
||||
const release = {};
|
||||
|
||||
return sceneElements.map((element) => {
|
||||
const actors = $(element).find('.actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
stars,
|
||||
},
|
||||
site,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
|
||||
const actors = $('.actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
const rawTags = $('.tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
const tags = await matchTags(rawTags);
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
description,
|
||||
actors,
|
||||
director,
|
||||
date,
|
||||
duration,
|
||||
tags,
|
||||
rating: {
|
||||
likes,
|
||||
dislikes,
|
||||
stars,
|
||||
},
|
||||
site,
|
||||
};
|
||||
console.log(release);
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await bhttp.get(`${site.url}/url`);
|
||||
const url = `${site.url}/${page}`;
|
||||
const qLatest = await geta(url, '.selector');
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchUpcoming(site) {
|
||||
const res = await bhttp.get(`${site.url}/url`);
|
||||
|
||||
return scrapeUpcoming(res.body.toString(), site);
|
||||
return qLatest && scrapeLatest(qLatest, site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await bhttp.get(url);
|
||||
const qScene = await get(url);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
return qScene && scrapeScene(qScene, site);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchUpcoming,
|
||||
fetchScene,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user