DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added 2 minute timeout to media fetch.

Browse Source
This commit is contained in:
ThePendulum
2020-04-26 04:21:57 +02:00
parent 2ac879d276
commit a223f933ce
2 changed files with 49 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/media.js
View File

@@ -365,7 +365,8 @@ async function fetchSource(source, baseMedia) {
}
// pipeline destroys streams
const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
// const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
const infoPromise = once(metaStream, 'info');
const metaPromise = type === 'image' ? metaStream.stats() : Promise.resolve();
await pipeline(
@@ -375,21 +376,7 @@ async function fetchSource(source, baseMedia) {
tempFileTarget,
);
/*
res.originalRes
.pipe(metaStream)
.pipe(hashStream)
.pipe(tempFileTarget);
*/
logger.silly(`Temporarily saved media from ${source.src}`);
const [stats, info] = await Promise.all([
metaPromise,
infoPromise,
]);
logger.silly(`Ended pipeline for ${source.src}`);
const [stats, info] = await Promise.all([metaPromise, infoPromise]);
hasher.end();
@@ -398,7 +385,7 @@ async function fetchSource(source, baseMedia) {
peakMemoryUsage = Math.max(getMemoryUsage(), peakMemoryUsage);
logger.silly(`Retrieved metadata from ${source.src}`);
logger.silly(`Fetched media from ${source.src}`);
return {
...source,
@@ -422,14 +409,21 @@ async function fetchSource(source, baseMedia) {
if (attempts < 3) {
await Promise.delay(1000);
return attempt(attempts + 1);
return Promise.race([
attempt(attempts + 1),
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch attempt ${attempts}/3 timed out, aborting ${source.src}`); }),
]);
}
throw new Error(`Failed to fetch ${source.src}: ${error.message}`);
}
}
return attempt(1);
return Promise.race([
attempt(1),
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch timed out, aborting ${source.src}`); }),
]);
}
async function trySource(baseSource, existingMedias, baseMedia) {
@@ -461,7 +455,13 @@ async function fetchMedia(baseMedia, existingMedias) {
try {
const source = await baseMedia.sources.reduce(
// try each source until success
(result, baseSource, baseSourceIndex) => result.catch(async () => trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)),
(result, baseSource, baseSourceIndex) => result.catch(async (error) => {
if (error.message) {
logger.warn(error.message);
}
return trySource(baseSource, existingMedias, baseMedia, baseSourceIndex);
}),
Promise.reject(new Error()),
);
@@ -599,7 +599,9 @@ async function associateReleaseMedia(releases) {
}, [])
.filter(Boolean);
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
if (associations.length > 0) {
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
}
}, Promise.resolve());
logger.debug(`Peak media fetching memory usage: ${peakMemoryUsage.toFixed(2)} MB`);