boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Major API change for 'q', renamed to 'qu', refactored modules. Fixed Gamma URL entry ID regex.

Browse Source
This commit is contained in:
ThePendulum
2020-03-09 02:02:29 +01:00
parent 7d71cf3a8c
commit 6cbb7f9c1e
21 changed files with 611 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/scrapers/nubiles.js
View File

@@ -18,13 +18,13 @@ async function getPhotos(albumUrl) {
}
function scrapeAll(scenes, site, origin) {
return scenes.map(({ q, qa, qu, qd }) => {
return scenes.map(({ qu }) => {
const release = {};
release.title = q('.title a', true);
release.title = qu.q('.title a', true);
const url = qu('.title a').split('?')[0];
const channelUrl = qu('.site-link');
const url = qu.url('.title a').split('?')[0];
const channelUrl = qu.url('.site-link');
if (/^http/.test(url)) {
const { pathname } = new URL(url);
@@ -39,74 +39,74 @@ function scrapeAll(scenes, site, origin) {
else if (site?.url) release.url = `${site.url}${url}`;
else if (origin) release.url = `${origin}${url}`;
} else {
release.entryId = q('a img', 'tube_tour_thumb_id');
release.entryId = qu.q('a img', 'tube_tour_thumb_id');
}
release.date = qd('.date', 'MMM D, YYYY');
release.actors = qa('.models a.model', true);
release.date = qu.date('.date', 'MMM D, YYYY');
release.actors = qu.all('.models a.model', true);
const poster = q('img').dataset.original;
const poster = qu.q('img').dataset.original;
release.poster = [
poster.replace('_640', '_1280'),
poster,
];
release.stars = Number(q('.rating', true));
release.likes = Number(q('.likes', true));
release.stars = Number(qu.q('.rating', true));
release.likes = Number(qu.q('.likes', true));
return release;
});
}
async function scrapeScene({ q, qa, qd, qp, qu, qi }, url, site) {
async function scrapeScene(qu, url, site) {
const release = {};
const { origin, pathname } = new URL(url);
release.url = `${origin}${pathname}`;
release.entryId = new URL(url).pathname.split('/')[3];
release.title = q('.content-pane-title h2', true);
release.description = q('.content-pane-column div', true);
release.title = qu.q('.content-pane-title h2', true);
release.description = qu.q('.content-pane-column div', true);
release.date = qd('.date', 'MMM D, YYYY');
release.date = qu.q('.date', 'MMM D, YYYY');
release.actors = qa('.content-pane-performers .model', true);
release.tags = qa('.categories a', true);
release.actors = qu.all('.content-pane-performers .model', true);
release.tags = qu.all('.categories a', true);
release.poster = qp() || qi('.fake-video-player img');
release.trailer = qa('source').map(source => ({
release.poster = qu.poster() || qu.img('.fake-video-player img');
release.trailer = qu.all('source').map(source => ({
src: source.src,
quality: Number(source.getAttribute('res')),
}));
release.stars = Number(q('.score', true));
release.likes = Number(q('#likecount', true));
release.stars = Number(qu.q('.score', true));
release.likes = Number(qu.q('#likecount', true));
const albumLink = qu('.content-pane-related-links a[href*="gallery"]');
const albumLink = qu.url('.content-pane-related-links a[href*="gallery"]');
if (albumLink) release.photos = await getPhotos(`${site.url}${albumLink}`);
return release;
}
function scrapeProfile({ q, qa, qi, qu }, _actorName, origin) {
function scrapeProfile({ qu }, _actorName, origin) {
const profile = {};
const keys = qa('.model-profile h5', true);
const values = qa('.model-profile h5 + p', true);
const keys = qu.all('.model-profile h5', true);
const values = qu.all('.model-profile h5 + p', true);
const bio = keys.reduce((acc, key, index) => ({ ...acc, [slugify(key, { delimiter: '_' })]: values[index] }), {});
profile.age = Number(bio.age);
profile.description = q('.model-bio', true);
profile.description = qu.q('.model-bio', true);
profile.residencePlace = bio.location;
profile.height = heightToCm(bio.height);
[profile.bust, profile.waist, profile.hip] = bio.figure.split('-').map(v => Number(v) || v);
profile.avatar = qi('.model-profile img');
profile.avatar = qu.img('.model-profile img');
const releases = qa('.content-grid-item').filter(el => /video\//.test(qu(el, '.img-wrapper a'))); // filter out photos
const releases = qu.all('.content-grid-item').filter(el => /video\//.test(qu.url(el, '.img-wrapper a'))); // filter out photos
profile.releases = scrapeAll(ctxa(releases), null, origin);
return profile;
@@ -145,7 +145,7 @@ async function fetchProfile(actorName, siteSlug) {
if (!resModels.ok) return resModels.status;
const modelPath = resModels.item.qa('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName));
const modelPath = resModels.item.qu.all('.content-grid-item a.title').find(el => slugify(el.textContent) === slugify(actorName));
if (modelPath) {
const modelUrl = `${origin}${modelPath}`;