DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Replaced queueMethod in media sources for new interval/concurrency options.

This commit is contained in:
DebaucheryLibrarian 2020-11-23 00:38:22 +01:00
parent aa1bba84aa
commit 8688c28d0f
3 changed files with 23 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/media.js
View File

@ -105,7 +105,8 @@ function toBaseSource(rawSource) {
if (rawSource.referer) baseSource.referer = rawSource.referer; if (rawSource.referer) baseSource.referer = rawSource.referer;
if (rawSource.host) baseSource.host = rawSource.host; if (rawSource.host) baseSource.host = rawSource.host;
if (rawSource.attempts) baseSource.attempts = rawSource.attempts; if (rawSource.attempts) baseSource.attempts = rawSource.attempts;
if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod; if (rawSource.interval) baseSource.interval = rawSource.interval;
if (rawSource.concurrency) baseSource.concurrency = rawSource.concurrency;
if (rawSource.credit !== undefined) baseSource.credit = rawSource.credit; if (rawSource.credit !== undefined) baseSource.credit = rawSource.credit;
if (rawSource.comment) baseSource.comment = rawSource.comment; if (rawSource.comment) baseSource.comment = rawSource.comment;
@ -435,7 +436,8 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
stream: true, // sources are fetched in parallel, don't gobble up memory stream: true, // sources are fetched in parallel, don't gobble up memory
transforms: [hashStream], transforms: [hashStream],
destination: tempFileTarget, destination: tempFileTarget,
queueMethod: source.queueMethod || null, // use http module's default ...(source.interval && { interval: source.interval }),
...(source.concurrency && { concurrency: source.concurrency }),
}); });
if (!res.ok) { if (!res.ok) {

8
src/scrapers/hookuphotshot.js
View File

@ -21,7 +21,7 @@ function scrapeAll(scenes) {
avatar: [ avatar: [
avatarEl.src.replace(/-\d+x\d+/, ''), avatarEl.src.replace(/-\d+x\d+/, ''),
avatarEl.src, avatarEl.src,
].map(src => ({ src, queueMethod: '1s' })), ].map(src => ({ src, interval: 1000, concurrency: 1 })),
}), }),
}; };
}).concat({ }).concat({
@ -37,12 +37,14 @@ function scrapeAll(scenes) {
release.poster = { release.poster = {
src: query.img('.date-img-swap'), src: query.img('.date-img-swap'),
queueMethod: '1s', interval: 1000,
concurrency: 1,
}; };
release.photos = [...Array(photoCount)].map((value, index) => ({ release.photos = [...Array(photoCount)].map((value, index) => ({
src: `${photoPath}/${String(index + 1).padStart(2, '0')}.jpg`, src: `${photoPath}/${String(index + 1).padStart(2, '0')}.jpg`,
queueMethod: '1s', interval: 1000,
concurrency: 1,
})); }));
// dates appear to be manually curated // dates appear to be manually curated

21
src/scrapers/whalemember.js
View File

@ -28,14 +28,16 @@ function scrapeLatest(html, site) {
src: /^http/.test(poster) ? poster : `https:${poster}`, src: /^http/.test(poster) ? poster : `https:${poster}`,
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
}; };
release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({ release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({
src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`), src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`),
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
})); }));
if (teaserEl) { if (teaserEl) {
@ -43,7 +45,8 @@ function scrapeLatest(html, site) {
src: teaserEl.dataset.src, src: teaserEl.dataset.src,
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
}; };
} }
@ -76,7 +79,8 @@ function scrapeScene(html, site, url) {
src: (/^http/.test(el.src) ? el.src : `https:${el.src}`), src: (/^http/.test(el.src) ? el.src : `https:${el.src}`),
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
})); }));
const posterEl = scene.querySelector('#no-player-image'); const posterEl = scene.querySelector('#no-player-image');
@ -88,14 +92,16 @@ function scrapeScene(html, site, url) {
src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`, src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`,
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
}; };
} else if (videoEl) { } else if (videoEl) {
release.poster = { release.poster = {
src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`, src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`,
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
}; };
} }
@ -104,7 +110,8 @@ function scrapeScene(html, site, url) {
src: trailerEl.src, src: trailerEl.src,
referer: site.url, referer: site.url,
attempts: 5, attempts: 5,
queueMethod: '5s', interval: 5000,
concurrency: 1,
}; };
} }