From c300d4d251ffb867a421be67e388416d8ab718bd Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Thu, 30 Jan 2020 03:53:54 +0100 Subject: [PATCH] Added trailer support to Vogov. Set preferred trailer resolution to 480p. Added 480p and other resolution trailers support to XEmpire. --- config/default.js | 1 + src/media.js | 4 +- src/scrapers/vogov.js | 126 +++++++++++++++++++++++++++++++++++++--- src/scrapers/xempire.js | 25 +++++++- 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/config/default.js b/config/default.js index b113d427..a26aaff5 100644 --- a/config/default.js +++ b/config/default.js @@ -44,6 +44,7 @@ module.exports = { path: './media', thumbnailSize: 320, // width for 16:9 will be exactly 576px thumbnailQuality: 100, + trailerQuality: 480, limit: 25, // max number of photos per release }, }; diff --git a/src/media.js b/src/media.js index ef042706..c7299eca 100644 --- a/src/media.js +++ b/src/media.js @@ -137,7 +137,7 @@ async function fetchPhoto(photoUrl, index, label, attempt = 1) { logger.warn(`Failed attempt ${attempt}/3 to fetch photo ${index + 1} for ${label} (${photoUrl}): ${error}`); if (attempt < 3) { - await Promise.delay(1000); + await Promise.delay(5000); return fetchPhoto(photoUrl, index, label, attempt + 1); } @@ -290,7 +290,7 @@ async function storeTrailer(trailers, { }, label) { // support scrapers supplying multiple qualities const trailer = Array.isArray(trailers) - ? trailers.find(trailerX => [1080, 720].includes(trailerX.quality)) || trailers[0] + ? trailers.find(trailerX => [config.media.trailerQuality, 480, 720, 1080].includes(trailerX.quality)) || trailers[0] : trailers; if (!trailer || !trailer.src) { diff --git a/src/scrapers/vogov.js b/src/scrapers/vogov.js index 1736c594..58f0fcc4 100644 --- a/src/scrapers/vogov.js +++ b/src/scrapers/vogov.js @@ -4,6 +4,93 @@ const bhttp = require('bhttp'); const { ex, ctxa } = require('../utils/q'); // const slugify = require('../utils/slugify'); +function getLicenseCode(html) { + const licensePrefix = 'license_code: \''; + const licenseStart = html.indexOf(licensePrefix); + const licenseCode = html.slice(licenseStart + licensePrefix.length, html.indexOf('\'', licenseStart + licensePrefix.length)); + + const c = '16px'; + let f; + let g; + let h; + let i; + let j; + let k; + let l; + let m; + let n; + + for (f = '', g = 1; g < licenseCode.length; g += 1) { + f += parseInt(licenseCode[g], 10) ? parseInt(licenseCode[g], 10) : 1; + } + + for (j = parseInt(f.length / 2, 10), + k = parseInt(f.substring(0, j + 1), 10), + l = parseInt(f.substring(j), 10), + g = l - k, + g < 0 && (g = -g), + f = g, + g = k - l, + g < 0 && (g = -g), + f += g, + f *= 2, + f = String(f), + i = (parseInt(c, 10) / 2) + 2, + m = '', + g = 0; g < j + 1; g += 1) { + for (h = 1; h <= 4; h += 1) { + n = parseInt(licenseCode[g + h], 10) + parseInt(f[g], 10); + + if (n >= i) n -= i; + m += n; + } + } + + return m; +} + +function decodeTrailerUrl(html, encodedTrailerUrl) { + const licenseCode = getLicenseCode(html); + const i = licenseCode; + + let j; + let k; + let l; + let m; + let n; + let o; + + const d = '16px'; + const g = encodedTrailerUrl.split('/').slice(2); + + let h = g[5].substring(0, 2 * parseInt(d, 10)); + + for (j = h, k = h.length - 1; k >= 0; k -= 1) { + for (l = k, m = k; m < i.length; m += 1) { + l += parseInt(i[m], 10); + } + + for (; l >= h.length;) { + l -= h.length; + } + + for (n = '', o = 0; o < h.length; o += 1) { + if (o === k) { + n += h[l]; + } else { + n += (o === l ? h[k] : h[o]); + } + } + + h = n; + } + + g[5] = g[5].replace(j, h); + const trailer = g.join('/'); + + return trailer; +} + function scrapeLatest(html) { const { document } = ex(html); @@ -52,15 +139,40 @@ function scrapeScene(html, url) { release.poster = html.slice(html.indexOf('http', previewStart), html.indexOf('.jpg', previewStart) + 4); } - /* - const trailerStart = html.indexOf('video_url'); - const trailerUrl = html.slice(html.indexOf('http', trailerStart), html.indexOf('.mp4', trailerStart) + 4); - const rnd = new Date().getTime(); + const varsPrefix = 'flashvars = {'; + const varsStart = html.indexOf(varsPrefix); + const varsString = html.slice(varsStart + varsPrefix.length, html.indexOf('};', varsStart)); - release.trailer = `${trailerUrl}/?rnd=${rnd}`; - */ + const vars = varsString.split(',').reduce((acc, item) => { + const [prop, value] = item.split(': '); + acc[prop.trim()] = value.trim().replace(/'/g, ''); + + return acc; + }, {}); + + release.trailer = [ + { + src: decodeTrailerUrl(html, vars.video_url), + quality: parseInt(vars.video_url_text, 10), + }, + { + src: decodeTrailerUrl(html, vars.video_alt_url), + quality: parseInt(vars.video_alt_url_text, 10), + }, + { + src: decodeTrailerUrl(html, vars.video_alt_url2), + quality: parseInt(vars.video_alt_url2_text, 10), + }, + { + src: decodeTrailerUrl(html, vars.video_alt_url3), + quality: parseInt(vars.video_alt_url3_text, 10), + }, + { + src: decodeTrailerUrl(html, vars.video_alt_url4), + quality: parseInt(vars.video_alt_url4_text, 10), + }, + ]; - // console.log(release); return release; } diff --git a/src/scrapers/xempire.js b/src/scrapers/xempire.js index f5c979b2..94bf5690 100644 --- a/src/scrapers/xempire.js +++ b/src/scrapers/xempire.js @@ -99,13 +99,34 @@ async function scrapeScene(html, url, site) { poster, photos, trailer: [ + { + src: trailer.replace('hd', 'sm'), + quality: 240, + }, + { + src: trailer.replace('hd', 'med'), + quality: 360, + }, + { + src: trailer.replace('hd', 'big'), + quality: 480, + }, + { + // probably 540p + src: trailer, + quality: parseInt(videoData.sizeOnLoad, 10), + }, + { + src: trailer.replace('hd', '720p'), + quality: 720, + }, { src: trailer.replace('hd', '1080p'), quality: 1080, }, { - src: trailer, - quality: parseInt(videoData.sizeOnLoad, 10), + src: trailer.replace('hd', '4k'), + quality: 2160, }, ], tags,