Replaced queueMethod in media sources for new interval/concurrency options.
This commit is contained in:
		
							parent
							
								
									aa1bba84aa
								
							
						
					
					
						commit
						8688c28d0f
					
				|  | @ -105,7 +105,8 @@ function toBaseSource(rawSource) { | |||
| 		if (rawSource.referer) baseSource.referer = rawSource.referer; | ||||
| 		if (rawSource.host) baseSource.host = rawSource.host; | ||||
| 		if (rawSource.attempts) baseSource.attempts = rawSource.attempts; | ||||
| 		if (rawSource.queueMethod) baseSource.queueMethod = rawSource.queueMethod; | ||||
| 		if (rawSource.interval) baseSource.interval = rawSource.interval; | ||||
| 		if (rawSource.concurrency) baseSource.concurrency = rawSource.concurrency; | ||||
| 
 | ||||
| 		if (rawSource.credit !== undefined) baseSource.credit = rawSource.credit; | ||||
| 		if (rawSource.comment) baseSource.comment = rawSource.comment; | ||||
|  | @ -435,7 +436,8 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) { | |||
| 		stream: true, // sources are fetched in parallel, don't gobble up memory
 | ||||
| 		transforms: [hashStream], | ||||
| 		destination: tempFileTarget, | ||||
| 		queueMethod: source.queueMethod || null, // use http module's default
 | ||||
| 		...(source.interval && { interval: source.interval }), | ||||
| 		...(source.concurrency && { concurrency: source.concurrency }), | ||||
| 	}); | ||||
| 
 | ||||
| 	if (!res.ok) { | ||||
|  |  | |||
|  | @ -21,7 +21,7 @@ function scrapeAll(scenes) { | |||
| 					avatar: [ | ||||
| 						avatarEl.src.replace(/-\d+x\d+/, ''), | ||||
| 						avatarEl.src, | ||||
| 					].map(src => ({ src, queueMethod: '1s' })), | ||||
| 					].map(src => ({ src, interval: 1000, concurrency: 1 })), | ||||
| 				}), | ||||
| 			}; | ||||
| 		}).concat({ | ||||
|  | @ -37,12 +37,14 @@ function scrapeAll(scenes) { | |||
| 
 | ||||
| 		release.poster = { | ||||
| 			src: query.img('.date-img-swap'), | ||||
| 			queueMethod: '1s', | ||||
| 			interval: 1000, | ||||
| 			concurrency: 1, | ||||
| 		}; | ||||
| 
 | ||||
| 		release.photos = [...Array(photoCount)].map((value, index) => ({ | ||||
| 			src: `${photoPath}/${String(index + 1).padStart(2, '0')}.jpg`, | ||||
| 			queueMethod: '1s', | ||||
| 			interval: 1000, | ||||
| 			concurrency: 1, | ||||
| 		})); | ||||
| 
 | ||||
| 		// dates appear to be manually curated
 | ||||
|  |  | |||
|  | @ -28,14 +28,16 @@ function scrapeLatest(html, site) { | |||
| 			src: /^http/.test(poster) ? poster : `https:${poster}`, | ||||
| 			referer: site.url, | ||||
| 			attempts: 5, | ||||
| 			queueMethod: '5s', | ||||
| 			interval: 5000, | ||||
| 			concurrency: 1, | ||||
| 		}; | ||||
| 
 | ||||
| 		release.photos = Array.from(scene.querySelectorAll('.rollover-thumbs img'), el => ({ | ||||
| 			src: (/^http/.test(el.dataset.src) ? el.dataset.src : `https:${el.dataset.src}`), | ||||
| 			referer: site.url, | ||||
| 			attempts: 5, | ||||
| 			queueMethod: '5s', | ||||
| 			interval: 5000, | ||||
| 			concurrency: 1, | ||||
| 		})); | ||||
| 
 | ||||
| 		if (teaserEl) { | ||||
|  | @ -43,7 +45,8 @@ function scrapeLatest(html, site) { | |||
| 				src: teaserEl.dataset.src, | ||||
| 				referer: site.url, | ||||
| 				attempts: 5, | ||||
| 				queueMethod: '5s', | ||||
| 				interval: 5000, | ||||
| 				concurrency: 1, | ||||
| 			}; | ||||
| 		} | ||||
| 
 | ||||
|  | @ -76,7 +79,8 @@ function scrapeScene(html, site, url) { | |||
| 		src: (/^http/.test(el.src) ? el.src : `https:${el.src}`), | ||||
| 		referer: site.url, | ||||
| 		attempts: 5, | ||||
| 		queueMethod: '5s', | ||||
| 		interval: 5000, | ||||
| 		concurrency: 1, | ||||
| 	})); | ||||
| 
 | ||||
| 	const posterEl = scene.querySelector('#no-player-image'); | ||||
|  | @ -88,14 +92,16 @@ function scrapeScene(html, site, url) { | |||
| 			src: /^http/.test(posterEl.src) ? posterEl.src : `https:${posterEl.src}`, | ||||
| 			referer: site.url, | ||||
| 			attempts: 5, | ||||
| 			queueMethod: '5s', | ||||
| 			interval: 5000, | ||||
| 			concurrency: 1, | ||||
| 		}; | ||||
| 	} else if (videoEl) { | ||||
| 		release.poster = { | ||||
| 			src: /^http/.test(videoEl.poster) ? videoEl.poster : `https:${videoEl.poster}`, | ||||
| 			referer: site.url, | ||||
| 			attempts: 5, | ||||
| 			queueMethod: '5s', | ||||
| 			interval: 5000, | ||||
| 			concurrency: 1, | ||||
| 		}; | ||||
| 	} | ||||
| 
 | ||||
|  | @ -104,7 +110,8 @@ function scrapeScene(html, site, url) { | |||
| 			src: trailerEl.src, | ||||
| 			referer: site.url, | ||||
| 			attempts: 5, | ||||
| 			queueMethod: '5s', | ||||
| 			interval: 5000, | ||||
| 			concurrency: 1, | ||||
| 		}; | ||||
| 	} | ||||
| 
 | ||||
|  |  | |||
		Loading…
	
		Reference in New Issue