DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Enabled pagination on network page.

Browse Source
This commit is contained in:
ThePendulum
2020-05-26 04:11:29 +02:00
parent fe69ec4175
commit 86377fec5f
43 changed files with 164 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/scrapers/bangbros.js
View File

@@ -16,7 +16,7 @@ function scrape(html, site) {
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.thmb_lnk');
const title = sceneLinkElement.attr('title');
const url = site.legacy
const url = site.parameters?.legacy
? `https://${site.url}{sceneLinkElement.attr('href')}`
: `https://bangbros.com${sceneLinkElement.attr('href')}`;
const shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
@@ -150,14 +150,14 @@ function scrapeSceneLegacy({ qu }, url) {
return release;
}
function scrapeProfile(html) {
function scrapeProfile(html, scope) {
const { q } = ex(html);
const profile = {};
const avatar = q('.profilePic img', 'src');
if (avatar) profile.avatar = `https:${avatar}`;
profile.releases = scrape(html);
profile.releases = scrape(html, scope.network);
return profile;
}
@@ -221,7 +221,7 @@ async function fetchScene(url, site, release) {
return scrapeScene(res.item.html, url, site);
}
async function fetchProfile(actorName) {
async function fetchProfile(actorName, scope) {
const actorSlug = slugify(actorName);
const url = `https://bangbros.com/search/${actorSlug}`;
const res = await bhttp.get(url);
@@ -233,7 +233,7 @@ async function fetchProfile(actorName) {
const actorRes = await bhttp.get(actorUrl);
if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString());
return scrapeProfile(actorRes.body.toString(), scope);
}
}
}

2
src/scrapers/brazzers.js
View File

@@ -100,7 +100,7 @@ async function scrapeScene(html, url, _site) {
const siteElement = $('.niche-site-logo');
// const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
release.channel = siteName.replace(/\s+/g, '').toLowerCase();
release.channel = slugify(siteName, '');
release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray();

15
src/scrapers/naughtyamerica.js
View File

@@ -59,16 +59,17 @@ function scrapeScene(html, url, site) {
const originalUrl = `${protocol}//${hostname}${pathname}`;
const entryId = originalUrl.split('-').slice(-1)[0];
const title = sceneElement.find('h1.scene-title.grey-text').text();
const title = sceneElement.find('h1.scene-title').text();
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
const date = moment.utc(sceneElement.find('span.entry-date').text(), 'MMM D, YYYY').toDate();
const actors = $('a.scene-title.grey-text.link').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const date = moment.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;
const poster = `https:${$('video, dl8-video').attr('poster')}`;
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => `https:${$(el).attr('href')}`).toArray();
const posterPath = $('video, dl8-video').attr('poster') || $('img.start-card').attr('src');
const poster = posterPath && `https:${posterPath}`;
const photos = $('.contain-scene-images.desktop-only a').map((index, el) => $(el).attr('href')).toArray().filter(Boolean).map(photo => `https:${photo}`);
const trailerEl = $('source');
const trailerSrc = trailerEl.attr('src');
@@ -90,10 +91,10 @@ function scrapeScene(html, url, site) {
tags,
photos,
poster,
trailer: {
trailer: trailerSrc ? {
src: trailerSrc,
type: trailerType,
},
} : null,
rating: null,
site,
channel,