From e896d5296854bca6b818cf444ac029ca2ac00e69 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Mon, 17 Aug 2020 15:53:20 +0200 Subject: [PATCH] Added separate task queue for video streams to prevent ffmpeg overstressing the CPU. Fixed entity parent in scene REST API. --- assets/components/container/container.vue | 2 ++ assets/components/footer/footer.vue | 8 +++++--- assets/components/home/home.vue | 14 +++++++++++--- config/default.js | 1 + seeds/02_sites.js | 2 +- src/argv.js | 5 ----- src/media.js | 10 +++++++--- src/releases.js | 8 ++++---- src/updates.js | 7 ------- 9 files changed, 31 insertions(+), 26 deletions(-) diff --git a/assets/components/container/container.vue b/assets/components/container/container.vue index e778e2b7..27e53252 100644 --- a/assets/components/container/container.vue +++ b/assets/components/container/container.vue @@ -88,6 +88,8 @@ export default { flex-grow: 1; overflow-y: auto; overflow-x: hidden; + box-shadow: 0 0 3px var(--shadow-weak); + z-index: 1; } .slide-enter-active, diff --git a/assets/components/footer/footer.vue b/assets/components/footer/footer.vue index 2a5d78c6..dc736090 100644 --- a/assets/components/footer/footer.vue +++ b/assets/components/footer/footer.vue @@ -11,10 +11,8 @@ diff --git a/assets/components/home/home.vue b/assets/components/home/home.vue index 6eb16349..3aee84d6 100644 --- a/assets/components/home/home.vue +++ b/assets/components/home/home.vue @@ -1,5 +1,5 @@ @@ -79,3 +79,11 @@ export default { }, }; + + diff --git a/config/default.js b/config/default.js index c858c413..c0abb69d 100644 --- a/config/default.js +++ b/config/default.js @@ -197,6 +197,7 @@ module.exports = { lazyQuality: 90, videoQuality: [480, 360, 320, 540, 720, 1080, 2160, 270, 240, 180], limit: 25, // max number of photos per release + streamConcurrency: 2, // max number of video streams (m3u8 etc.) to fetch and process at once }, titleSlugLength: 50, }; diff --git a/seeds/02_sites.js b/seeds/02_sites.js index ed6dcf75..8aeef7ef 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -4968,7 +4968,7 @@ const sites = [ }, }, { - name: 'Horny doctor', + name: 'Horny Doctor', slug: 'hornydoctor', url: 'https://www.hornydoctor.com', parent: 'porncz', diff --git a/src/argv.js b/src/argv.js index 413e87e5..d7560590 100644 --- a/src/argv.js +++ b/src/argv.js @@ -30,11 +30,6 @@ const { argv } = yargs type: 'boolean', alias: 'web', }) - .option('all', { - describe: 'Scrape channels and networks defined in configuration', - type: 'boolean', - alias: 'scrape', - }) .option('networks', { describe: 'Network to scrape all channels from (overrides configuration)', type: 'array', diff --git a/src/media.js b/src/media.js index 5cd19f2f..f044fbc0 100644 --- a/src/media.js +++ b/src/media.js @@ -13,6 +13,7 @@ const mime = require('mime'); const ffmpeg = require('fluent-ffmpeg'); const sharp = require('sharp'); const blake2 = require('blake2'); +const taskQueue = require('promise-task-queue'); const logger = require('./logger')(__filename); const argv = require('./argv'); @@ -22,6 +23,7 @@ const bulkInsert = require('./utils/bulk-insert'); const { get } = require('./utils/qu'); const pipeline = util.promisify(stream.pipeline); +const streamQueue = taskQueue(); function sampleMedias(medias, limit = config.media.limit, preferLast = true) { // limit media sets, use extras as fallbacks @@ -419,7 +421,7 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) { }; } -async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) { +streamQueue.define('fetchStreamSource', async ({ source, tempFileTarget, hashStream }) => { const meta = { mimetype: 'video/mp4' }; const video = ffmpeg(source.stream) @@ -434,7 +436,9 @@ async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStrea logger.verbose(`Finished fetching stream from ${source.stream}`); return meta; -} +}, { + concurrency: config.media.streamConcurrency, +}); async function fetchSource(source, baseMedia) { logger.silly(`Fetching media from ${source.src}`); @@ -457,7 +461,7 @@ async function fetchSource(source, baseMedia) { }); const { mimetype } = source.stream - ? await fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) + ? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream }) : await fetchHttpSource(source, tempFileTarget, hashStream); hasher.end(); diff --git a/src/releases.js b/src/releases.js index ed40e52b..471328f2 100644 --- a/src/releases.js +++ b/src/releases.js @@ -20,10 +20,10 @@ function curateRelease(release, withMedia = false) { id: release.entity.id, name: release.entity.name, slug: release.entity.slug, - parent: release.entity.parent && { - id: release.entity.parent.id, - name: release.entity.parent.name, - slug: release.entity.parent.slug, + parent: release.parent && { + id: release.parent.id, + name: release.parent.name, + slug: release.parent.slug, }, }, actors: (release.actors || []).map(actor => ({ diff --git a/src/updates.js b/src/updates.js index ed50bf7e..c8c4ab7d 100644 --- a/src/updates.js +++ b/src/updates.js @@ -74,13 +74,6 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false) { return accReleases; } - /* prevents argv.last from being satisfied - if (latestReleases.length === 0) { - // scraper successfully requested releases, but found none - return accReleases; - } - */ - const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity: release.entity || entity, // allow override