boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Allowing image sources to specify queue method. Using 5s queue for Whale Member to avoid CDN time-outs.

Browse Source
This commit is contained in:
ThePendulum
2020-07-01 04:47:05 +02:00
parent 53870fda89
commit 1f444e58ce
12 changed files with 183 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/scrapers/whalemember.js
View File

@@ -19,13 +19,32 @@ function scrapeLatest(html, site) {
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), el => el.textContent);
// slow CDN?
const poster = scene.querySelector('.single-image').dataset.src;
release.poster = /^http/.test(poster) ? poster : `https:${poster}`;
const teaserEl = scene.querySelector('source');
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`));
release.poster = {
src: /^http/.test(poster) ? poster : `https:${poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
const trailerEl = scene.querySelector('source');
if (trailerEl) release.trailer = { src: trailerEl.dataset.src };
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
if (teaserEl) {
release.teaser = {
src: teaserEl.dataset.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release;
});
@@ -51,16 +70,42 @@ function scrapeScene(html, site, url) {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
}
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => (/^http/.test(el.src) ? el.src : `https:${el.src}`));
// unreliable CDN
release.photos = Array.from(scene.querySelectorAll('#t2019-main .t2019-thumbs img'), el => ({
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url,
attempts: 5,
queueMethod: '5s',
}));
const posterEl = scene.querySelector('#no-player-image');
const videoEl = scene.querySelector('video');
if (posterEl) release.poster = /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`;
else if (videoEl) release.poster = /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`;
const trailerEl = scene.querySelector('#t2019-video source');
if (trailerEl) release.trailer = { src: trailerEl.src };
if (posterEl) {
release.poster = {
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
} else if (videoEl) {
release.poster = {
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
if (trailerEl) {
release.trailer = {
src: trailerEl.src,
referer: site.url,
attempts: 5,
queueMethod: '5s',
};
}
return release;
}