From a223f933cecbbd724d4183c5f172a2db281fd902 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sun, 26 Apr 2020 04:21:57 +0200 Subject: [PATCH] Added 2 minute timeout to media fetch. --- src/media.js | 44 +++++++++++++++++++----------------- src/scrapers/vixen.js | 52 +++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/media.js b/src/media.js index 64fe9efd..5588fd34 100644 --- a/src/media.js +++ b/src/media.js @@ -365,7 +365,8 @@ async function fetchSource(source, baseMedia) { } // pipeline destroys streams - const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]); + // const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]); + const infoPromise = once(metaStream, 'info'); const metaPromise = type === 'image' ? metaStream.stats() : Promise.resolve(); await pipeline( @@ -375,21 +376,7 @@ async function fetchSource(source, baseMedia) { tempFileTarget, ); - /* - res.originalRes - .pipe(metaStream) - .pipe(hashStream) - .pipe(tempFileTarget); - */ - - logger.silly(`Temporarily saved media from ${source.src}`); - - const [stats, info] = await Promise.all([ - metaPromise, - infoPromise, - ]); - - logger.silly(`Ended pipeline for ${source.src}`); + const [stats, info] = await Promise.all([metaPromise, infoPromise]); hasher.end(); @@ -398,7 +385,7 @@ async function fetchSource(source, baseMedia) { peakMemoryUsage = Math.max(getMemoryUsage(), peakMemoryUsage); - logger.silly(`Retrieved metadata from ${source.src}`); + logger.silly(`Fetched media from ${source.src}`); return { ...source, @@ -422,14 +409,21 @@ async function fetchSource(source, baseMedia) { if (attempts < 3) { await Promise.delay(1000); - return attempt(attempts + 1); + + return Promise.race([ + attempt(attempts + 1), + Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch attempt ${attempts}/3 timed out, aborting ${source.src}`); }), + ]); } throw new Error(`Failed to fetch ${source.src}: ${error.message}`); } } - return attempt(1); + return Promise.race([ + attempt(1), + Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch timed out, aborting ${source.src}`); }), + ]); } async function trySource(baseSource, existingMedias, baseMedia) { @@ -461,7 +455,13 @@ async function fetchMedia(baseMedia, existingMedias) { try { const source = await baseMedia.sources.reduce( // try each source until success - (result, baseSource, baseSourceIndex) => result.catch(async () => trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)), + (result, baseSource, baseSourceIndex) => result.catch(async (error) => { + if (error.message) { + logger.warn(error.message); + } + + return trySource(baseSource, existingMedias, baseMedia, baseSourceIndex); + }), Promise.reject(new Error()), ); @@ -599,7 +599,9 @@ async function associateReleaseMedia(releases) { }, []) .filter(Boolean); - await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`); + if (associations.length > 0) { + await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`); + } }, Promise.resolve()); logger.debug(`Peak media fetching memory usage: ${peakMemoryUsage.toFixed(2)} MB`); diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js index 33a08fa9..ddbd8bc1 100644 --- a/src/scrapers/vixen.js +++ b/src/scrapers/vixen.js @@ -42,6 +42,32 @@ function getAvatarFallbacks(avatar) { .flat(); } +async function getTrailer(scene, site, url) { + const qualities = [360, 480, 720, 1080, 2160]; + + const tokenRes = await post(`${site.url}/api/__record_tknreq`, { + file: scene.previewVideoUrl1080P, + sizes: qualities.join('+'), + type: 'trailer', + }, { referer: url }); + + if (!tokenRes.ok) { + return null; + } + + const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`; + const trailersRes = await post(trailerUrl, null, { referer: url }); + + if (trailersRes.ok) { + return qualities.map(quality => (trailersRes.body[quality] ? { + src: trailersRes.body[quality].token, + quality, + } : null)).filter(Boolean); + } + + return null; +} + function scrapeAll(scenes, site, origin) { return scenes.map((scene) => { const release = {}; @@ -90,32 +116,6 @@ function scrapeUpcoming(scene, site) { return [release]; } -async function getTrailer(scene, site, url) { - const qualities = [360, 480, 720, 1080, 2160]; - - const tokenRes = await post(`${site.url}/api/__record_tknreq`, { - file: scene.previewVideoUrl1080P, - sizes: qualities.join('+'), - type: 'trailer', - }, { referer: url }); - - if (!tokenRes.ok) { - return null; - } - - const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`; - const trailersRes = await post(trailerUrl, null, { referer: url }); - - if (trailersRes.ok) { - return qualities.map(quality => (trailersRes.body[quality] ? { - src: trailersRes.body[quality].token, - quality, - } : null)).filter(Boolean); - } - - return null; -} - async function scrapeScene(data, url, site, baseRelease) { const scene = data.video;