diff --git a/config/default.js b/config/default.js index c96c49a6..7165b2b1 100644 --- a/config/default.js +++ b/config/default.js @@ -139,6 +139,19 @@ module.exports = { 'freeones', 'freeonesLegacy', ], + proxy: { + enable: false, + host: '', + port: 8888, + hostnames: [ + 'www.vixen.com', + 'www.blacked.com', + 'www.blackedraw.com', + 'www.tushy.com', + 'www.tushyraw.com', + 'www.deeper.com', + ], + }, fetchAfter: [1, 'week'], nullDateLimit: 3, media: { diff --git a/package-lock.json b/package-lock.json index 8829469f..206458f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11050,6 +11050,11 @@ } } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index 775519d4..566d8834 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "template-format": "^1.2.5", "tough-cookie": "^3.0.1", "tty-table": "^2.8.12", + "tunnel": "0.0.6", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "vue": "^2.6.11", diff --git a/public/img/logos/gamma/misc/powered.jpg b/public/img/logos/gamma/misc/powered.jpg new file mode 100644 index 00000000..9c1380b3 Binary files /dev/null and b/public/img/logos/gamma/misc/powered.jpg differ diff --git a/src/app.js b/src/app.js index a393df29..6309fcbd 100644 --- a/src/app.js +++ b/src/app.js @@ -8,6 +8,7 @@ const knex = require('./knex'); const fetchUpdates = require('./updates'); const fetchDeep = require('./deep'); const { storeReleases } = require('./store-releases'); +const { updateReleasesSearch } = require('./releases'); // const { storeReleaseActors } = require('./actors'); async function init() { @@ -16,6 +17,12 @@ async function init() { return; } + if (argv.updateSearch) { + await updateReleasesSearch(); + knex.destroy(); + return; + } + const updateBaseReleases = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates(); const updateDeepReleases = updateBaseReleases && await fetchDeep(updateBaseReleases); diff --git a/src/releases.js b/src/releases.js index ae5dedb6..0b13c77c 100644 --- a/src/releases.js +++ b/src/releases.js @@ -368,8 +368,10 @@ async function updateReleasesSearch(releaseIds) { releases.title || ' ' || networks.name || ' ' || networks.slug || ' ' || + networks.url || ' ' || sites.name || ' ' || sites.slug || ' ' || + COALESCE(sites.url, '') || ' ' || COALESCE(sites.alias, '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' || COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' || @@ -386,7 +388,7 @@ async function updateReleasesSearch(releaseIds) { LEFT JOIN tags ON local_tags.tag_id = tags.id LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} - GROUP BY releases.id, sites.name, sites.slug, sites.alias, networks.name, networks.slug; + GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url; `, releaseIds && [releaseIds]); if (documents.rows?.length > 0) { diff --git a/src/utils/http.js b/src/utils/http.js index ed7398d0..067839ae 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -1,10 +1,28 @@ 'use strict'; +const config = require('config'); +const tunnel = require('tunnel'); const bhttp = require('bhttp'); const taskQueue = require('promise-task-queue'); const logger = require('../logger')(__filename); +const proxyAgent = tunnel.httpsOverHttp({ + proxy: { + host: config.proxy.host, + port: config.proxy.port, + }, +}); + +function useProxy(url) { + if (!config.proxy.enable) { + return false; + } + + const { hostname } = new URL(url); + return config.proxy.hostnames.includes(hostname); +} + const queue = taskQueue(); queue.on('concurrencyReached:httpGet', () => { @@ -15,47 +33,42 @@ queue.on('concurrencyReached:httpPost', () => { logger.silly('Queueing POST requests'); }); -queue.define('httpGet', async ({ - url, - timeout = 30000, - options = {}, -}) => { - logger.silly(`GET ${url}`); - - const res = await bhttp.get(url, { - responseTimeout: timeout, - ...options, - }); - - res.code = res.statusCode; - - return res; -}, { - concurrency: 20, -}); - -queue.define('httpPost', async ({ +queue.define('http', async ({ url, + method = 'GET', body, timeout = 30000, options = {}, }) => { - logger.silly(`POST ${url} with ${body}`); + if (body) { + logger.silly(`${method.toUpperCase()} ${url} with ${body}`); + } else { + logger.silly(`${method.toUpperCase()} ${url}`); + } - const res = await bhttp.post(url, body, { + const reqOptions = { responseTimeout: timeout, ...options, - }); + }; - res.code = res.statusCode; + if (useProxy(url)) { + reqOptions.agent = proxyAgent; + } - return res; + const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase()) + ? await bhttp[method.toLowerCase()](url, body, reqOptions) + : await bhttp[method.toLowerCase()](url, reqOptions); + + return { + ...res, + code: res.statusCode, + }; }, { concurrency: 20, }); async function get(url, options) { - return queue.push('httpGet', { + return queue.push('http', { method: 'get', url, options, @@ -63,7 +76,7 @@ async function get(url, options) { } async function post(url, body, options) { - return queue.push('httpPost', { + return queue.push('http', { url, body, options, diff --git a/src/utils/qu.js b/src/utils/qu.js index 1909a387..42691b02 100644 --- a/src/utils/qu.js +++ b/src/utils/qu.js @@ -287,7 +287,7 @@ function extractAll(htmlValue, selector) { return initAll(window.document, selector, window); } -async function get(urlValue, selector, headers, queryAll = false) { +async function get(urlValue, selector, headers, options, queryAll = false) { const res = await http.get(urlValue, { headers, }); @@ -315,8 +315,8 @@ async function get(urlValue, selector, headers, queryAll = false) { }; } -async function getAll(urlValue, selector, headers) { - return get(urlValue, selector, headers, true); +async function getAll(urlValue, selector, headers, options) { + return get(urlValue, selector, headers, options, true); } module.exports = {