DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added 2 minute timeout to media fetch.

This commit is contained in:
ThePendulum 2020-04-26 04:21:57 +02:00
parent 2ac879d276
commit a223f933ce
2 changed files with 49 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/media.js
View File

@ -365,7 +365,8 @@ async function fetchSource(source, baseMedia) {
}
// pipeline destroys streams
const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
// const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
const infoPromise = once(metaStream, 'info');
const metaPromise = type === 'image' ? metaStream.stats() : Promise.resolve();
await pipeline(
@ -375,21 +376,7 @@ async function fetchSource(source, baseMedia) {
tempFileTarget,
);
/*
res.originalRes
.pipe(metaStream)
.pipe(hashStream)
.pipe(tempFileTarget);
*/
logger.silly(`Temporarily saved media from ${source.src}`);
const [stats, info] = await Promise.all([
metaPromise,
infoPromise,
]);
logger.silly(`Ended pipeline for ${source.src}`);
const [stats, info] = await Promise.all([metaPromise, infoPromise]);
hasher.end();
@ -398,7 +385,7 @@ async function fetchSource(source, baseMedia) {
peakMemoryUsage = Math.max(getMemoryUsage(), peakMemoryUsage);
logger.silly(`Retrieved metadata from ${source.src}`);
logger.silly(`Fetched media from ${source.src}`);
return {
...source,
@ -422,14 +409,21 @@ async function fetchSource(source, baseMedia) {
if (attempts < 3) {
await Promise.delay(1000);
return attempt(attempts + 1);
return Promise.race([
attempt(attempts + 1),
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch attempt ${attempts}/3 timed out, aborting ${source.src}`); }),
]);
}
throw new Error(`Failed to fetch ${source.src}: ${error.message}`);
}
}
return attempt(1);
return Promise.race([
attempt(1),
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch timed out, aborting ${source.src}`); }),
]);
}
async function trySource(baseSource, existingMedias, baseMedia) {
@ -461,7 +455,13 @@ async function fetchMedia(baseMedia, existingMedias) {
try {
const source = await baseMedia.sources.reduce(
// try each source until success
(result, baseSource, baseSourceIndex) => result.catch(async () => trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)),
(result, baseSource, baseSourceIndex) => result.catch(async (error) => {
if (error.message) {
logger.warn(error.message);
}
return trySource(baseSource, existingMedias, baseMedia, baseSourceIndex);
}),
Promise.reject(new Error()),
);
@ -599,7 +599,9 @@ async function associateReleaseMedia(releases) {
}, [])
.filter(Boolean);
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
if (associations.length > 0) {
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
}
}, Promise.resolve());
logger.debug(`Peak media fetching memory usage: ${peakMemoryUsage.toFixed(2)} MB`);

52
src/scrapers/vixen.js
View File

@ -42,6 +42,32 @@ function getAvatarFallbacks(avatar) {
.flat();
}
async function getTrailer(scene, site, url) {
const qualities = [360, 480, 720, 1080, 2160];
const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
file: scene.previewVideoUrl1080P,
sizes: qualities.join('+'),
type: 'trailer',
}, { referer: url });
if (!tokenRes.ok) {
return null;
}
const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
const trailersRes = await post(trailerUrl, null, { referer: url });
if (trailersRes.ok) {
return qualities.map(quality => (trailersRes.body[quality] ? {
src: trailersRes.body[quality].token,
quality,
} : null)).filter(Boolean);
}
return null;
}
function scrapeAll(scenes, site, origin) {
return scenes.map((scene) => {
const release = {};
@ -90,32 +116,6 @@ function scrapeUpcoming(scene, site) {
return [release];
}
async function getTrailer(scene, site, url) {
const qualities = [360, 480, 720, 1080, 2160];
const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
file: scene.previewVideoUrl1080P,
sizes: qualities.join('+'),
type: 'trailer',
}, { referer: url });
if (!tokenRes.ok) {
return null;
}
const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
const trailersRes = await post(trailerUrl, null, { referer: url });
if (trailersRes.ok) {
return qualities.map(quality => (trailersRes.body[quality] ? {
src: trailersRes.body[quality].token,
quality,
} : null)).filter(Boolean);
}
return null;
}
async function scrapeScene(data, url, site, baseRelease) {
const scene = data.video;