DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Major API change for 'q', renamed to 'qu', refactored modules. Fixed Gamma URL entry ID regex.

Browse Source
This commit is contained in:
ThePendulum
2020-03-09 02:02:29 +01:00
parent 7d71cf3a8c
commit 6cbb7f9c1e
21 changed files with 611 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/scrapers/score.js
View File

@@ -65,49 +65,49 @@ function scrapeAll(html, site) {
}
async function scrapeScene(html, url, site) {
const { q, qa, qtext, qi, qd, ql, qu, qis, qp } = ex(html, '#videos-page, #content');
const { qu } = ex(html, '#videos-page, #content');
const release = {};
[release.entryId] = new URL(url).pathname.split('/').slice(-2);
release.title = q('h2.text-uppercase, h2.title, #breadcrumb-top + h1', true)
|| q('h1.m-title', true)?.split(/»|\//).slice(-1)[0].trim();
release.description = qtext('.p-desc, .desc');
release.title = qu.q('h2.text-uppercase, h2.title, #breadcrumb-top + h1', true)
|| qu.q('h1.m-title', true)?.split(/»|\//).slice(-1)[0].trim();
release.description = qu.text('.p-desc, .desc');
release.actors = qa('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
release.actors = qu.all('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true);
if (release.actors.length === 0) {
const actorEl = qa('.stat').find(stat => /Featuring/.test(stat.textContent));
const actorString = qtext(actorEl);
const actorEl = qu.all('.stat').find(stat => /Featuring/.test(stat.textContent));
const actorString = qu.text(actorEl);
release.actors = actorString?.split(/,\band\b|,/g).map(actor => actor.trim()) || [];
}
if (release.actors.length === 0 && site.parameters?.actors) release.actors = site.parameters.actors;
release.tags = qa('a[href*=tag]', true);
release.tags = qu.all('a[href*=tag]', true);
const dateEl = qa('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent));
release.date = qd(dateEl, null, 'MMMM Do, YYYY')
|| qd('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|| qd('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
const dateEl = qu.all('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent));
release.date = qu.date(dateEl, null, 'MMMM Do, YYYY')
|| qu.date('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/)
|| qu.date('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/);
const durationEl = qa('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent));
release.duration = ql(durationEl);
const durationEl = qu.all('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent));
release.duration = qu.dur(durationEl);
release.poster = qp('video') || qi('.flowplayer img') || qi('img'); // _800.jpg is larger than _xl.jpg in landscape
const photosUrl = qu('.stat a[href*=photos]');
release.poster = qu.poster('video') || qu.img('.flowplayer img') || qu.img('img'); // _800.jpg is larger than _xl.jpg in landscape
const photosUrl = qu.url('.stat a[href*=photos]');
if (photosUrl) {
release.photos = await fetchPhotos(photosUrl);
} else {
release.photos = qis('img[src*=ThumbNails], .p-photos .tn img').map(photo => [
release.photos = qu.imgs('img[src*=ThumbNails], .p-photos .tn img').map(photo => [
photo.replace('_tn', ''),
photo,
]);
}
const trailers = qa('a[href*=Trailers]');
const trailers = qu.all('a[href*=Trailers]');
if (trailers) {
release.trailer = trailers.map((trailer) => {
@@ -119,7 +119,7 @@ async function scrapeScene(html, url, site) {
}).filter(Boolean);
}
const stars = q('.rate-box').dataset.score;
const stars = qu.q('.rate-box').dataset.score;
if (stars) release.rating = { stars };
return release;
@@ -133,11 +133,11 @@ function scrapeModels(html, actorName) {
}
async function fetchActorReleases(url, accReleases = []) {
const { document, qu } = await get(url);
const res = await get(url);
if (document) {
const releases = accReleases.concat(scrapeAll(document.body.outerHTML));
const nextPage = qu('.next-pg');
if (res.ok) {
const releases = accReleases.concat(scrapeAll(res.item.document.body.outerHTML));
const nextPage = res.item.qu.url('.next-pg');
if (nextPage && new URL(nextPage).searchParams.has('page')) { // last page has 'next' button linking to join page
return fetchActorReleases(nextPage, releases);