DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Major API change for 'q', renamed to 'qu', refactored modules. Fixed Gamma URL entry ID regex.

Browse Source
This commit is contained in:
ThePendulum
2020-03-09 02:02:29 +01:00
parent 7d71cf3a8c
commit 6cbb7f9c1e
21 changed files with 611 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/scrapers/julesjordan.js
View File

@@ -118,25 +118,41 @@ async function getPhotos(entryId, site, type = 'highres', page = 1) {
return getPhotosLegacy(entryId, site, 'highres', 1);
}
function getEntryId(html) {
const entryId = html.match(/showtagform\((\d+)\)/);
if (entryId) {
return entryId[1];
}
const setIdIndex = html.indexOf('setid:"');
if (setIdIndex) {
return html.slice(setIdIndex, html.indexOf(',', setIdIndex)).match(/\d+/)[0];
}
return null;
}
function scrapeAll(scenes, site) {
return scenes.map(({ el, q, qa, qh, qu, qd, qi, qis }) => {
return scenes.map(({ qu }) => {
const release = {};
release.entryId = el.dataset.setid || q('.rating_box')?.dataset.id;
release.entryId = qu.el.dataset.setid || qu.q('.rating_box')?.dataset.id;
release.url = qu('.update_title, .dvd_info > a, a ~ a');
release.title = q('.update_title, .dvd_info > a, a ~ a', true);
release.date = qd('.update_date', 'MM/DD/YYYY');
release.url = qu.url('.update_title, .dvd_info > a, a ~ a');
release.title = qu.q('.update_title, .dvd_info > a, a ~ a', true);
release.date = qu.date('.update_date', 'MM/DD/YYYY');
release.actors = qa('.update_models a', true);
release.actors = qu.all('.update_models a', true);
const dvdPhotos = qis('.dvd_preview_thumb');
const photoCount = Number(q('a img.thumbs', 'cnt')) || 1;
const dvdPhotos = qu.imgs('.dvd_preview_thumb');
const photoCount = Number(qu.q('a img.thumbs', 'cnt')) || 1;
[release.poster, ...release.photos] = dvdPhotos.length
? dvdPhotos
: Array.from({ length: photoCount }).map((value, index) => {
const src = qi('a img.thumbs', `src${index}_1x`) || qi('a img.thumbs', `src${index}`) || qi('a img.thumbs');
const src = qu.img('a img.thumbs', `src${index}_1x`) || qu.img('a img.thumbs', `src${index}`) || qu.img('a img.thumbs');
return src ? {
src: /^http/.test(src) ? src : `${site.url}${src}`,
@@ -144,7 +160,7 @@ function scrapeAll(scenes, site) {
} : null;
}).filter(Boolean);
const teaserScript = qh('script');
const teaserScript = qu.content('script');
if (teaserScript) {
const src = teaserScript.slice(teaserScript.indexOf('http'), teaserScript.indexOf('.mp4') + 4);
if (src) release.teaser = { src };
@@ -204,50 +220,17 @@ function scrapeUpcoming(html, site) {
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
async function scrapeScene({ qu }, url, site) {
const release = { url, site };
release.title = $('.title_bar_hilite').text().trim();
release.entryId = getEntryId(qu.html);
release.title = qu.q('.title_bar_hilite', true);
release.description = qu.q('.update_description', true);
const entryId = html.match(/showtagform\((\d+)\)/);
release.date = qu.date('.update_date', 'MM/DD/YYYY', null, 'innerHTML');
release.actors = qu.all('.backgroundcolor_info > .update_models a, .item .update_models a', true);
if (entryId) release.entryId = entryId[1];
else {
const setIdIndex = html.indexOf('setid:"');
if (setIdIndex) release.entryId = html.slice(setIdIndex, html.indexOf(',', setIdIndex)).match(/\d+/)[0];
}
const dateElement = $('.update_date').text().trim();
const dateComment = $('*')
.contents()
.toArray()
.find(({ type, data }) => type === 'comment' && data.match('Date OFF'));
if (dateElement) {
release.date = moment
.utc($('.update_date').text(), 'MM/DD/YYYY')
.toDate();
}
if (dateComment) {
release.date = moment
.utc(dateComment.nodeValue.match(/\d{2}\/\d{2}\/\d{4}/), 'MM/DD/YYYY')
.toDate();
}
release.description = $('.update_description').text().trim();
release.actors = $('.backgroundcolor_info > .update_models a, .item .update_models a')
.map((_actorIndex, actorElement) => $(actorElement).text())
.toArray();
const infoLines = $('script:contains("useimage")')
.html()
.split('\n');
const posterPath = infoLines.find(line => line.match('useimage')).replace('useimage = "', '').slice(0, -2);
const posterPath = qu.html.match(/useimage = "(.*)"/)?.[1];
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`;
@@ -261,7 +244,7 @@ async function scrapeScene(html, url, site) {
}
if (site.slug !== 'manuelferrara') {
const trailerLines = infoLines.filter(line => /movie\["Trailer\w*"\]\[/.test(line));
const trailerLines = qu.html.split('\n').filter(line => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) {
release.trailer = trailerLines.map((trailerLine) => {
@@ -270,19 +253,24 @@ async function scrapeScene(html, url, site) {
return src && {
src: /^http/.test(src) ? src : `${site.url}${src}`,
quality: quality && Number(quality),
quality: quality && Number(quality.replace('558', '540')),
};
}).filter(Boolean);
}
}
release.photos = await getPhotos(release.entryId, site);
release.tags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.tags = qu.all('.update_tags a', true);
const movie = $('.update_dvds a').attr('href');
if (movie) release.movie = movie;
if (qu.exists('.update_dvds a')) {
release.movie = {
url: qu.url('.update_dvds a'),
title: qu.q('.update_dvds a', true),
};
}
release.stars = Number($('.avg_rating').text().trim().replace(/[\s|Avg Rating:]/g, ''));
const stars = Number(qu.q('.avg_rating', true)?.replace(/[\s|Avg Rating:]/g, ''));
if (stars) release.stars = stars;
return release;
}
@@ -371,9 +359,9 @@ async function fetchUpcoming(site) {
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
const res = await get(url);
return scrapeScene(res.body.toString(), url, site);
return res.ok ? scrapeScene(res.item, url, site) : res.status;
}
async function fetchMovie(url, site) {