Added configurable proxy to HTTP module (also used by qu). Added network and site URL to search documents.

This commit is contained in:
ThePendulum 2020-03-19 01:54:25 +01:00
parent e4b269956e
commit 4b310e9dfa
8 changed files with 72 additions and 31 deletions

View File

@ -139,6 +139,19 @@ module.exports = {
'freeones', 'freeones',
'freeonesLegacy', 'freeonesLegacy',
], ],
proxy: {
enable: false,
host: '',
port: 8888,
hostnames: [
'www.vixen.com',
'www.blacked.com',
'www.blackedraw.com',
'www.tushy.com',
'www.tushyraw.com',
'www.deeper.com',
],
},
fetchAfter: [1, 'week'], fetchAfter: [1, 'week'],
nullDateLimit: 3, nullDateLimit: 3,
media: { media: {

5
package-lock.json generated
View File

@ -11050,6 +11050,11 @@
} }
} }
}, },
"tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
},
"tunnel-agent": { "tunnel-agent": {
"version": "0.6.0", "version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",

View File

@ -110,6 +110,7 @@
"template-format": "^1.2.5", "template-format": "^1.2.5",
"tough-cookie": "^3.0.1", "tough-cookie": "^3.0.1",
"tty-table": "^2.8.12", "tty-table": "^2.8.12",
"tunnel": "0.0.6",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3", "v-tooltip": "^2.0.3",
"vue": "^2.6.11", "vue": "^2.6.11",

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@ -8,6 +8,7 @@ const knex = require('./knex');
const fetchUpdates = require('./updates'); const fetchUpdates = require('./updates');
const fetchDeep = require('./deep'); const fetchDeep = require('./deep');
const { storeReleases } = require('./store-releases'); const { storeReleases } = require('./store-releases');
const { updateReleasesSearch } = require('./releases');
// const { storeReleaseActors } = require('./actors'); // const { storeReleaseActors } = require('./actors');
async function init() { async function init() {
@ -16,6 +17,12 @@ async function init() {
return; return;
} }
if (argv.updateSearch) {
await updateReleasesSearch();
knex.destroy();
return;
}
const updateBaseReleases = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates(); const updateBaseReleases = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates();
const updateDeepReleases = updateBaseReleases && await fetchDeep(updateBaseReleases); const updateDeepReleases = updateBaseReleases && await fetchDeep(updateBaseReleases);

View File

@ -368,8 +368,10 @@ async function updateReleasesSearch(releaseIds) {
releases.title || ' ' || releases.title || ' ' ||
networks.name || ' ' || networks.name || ' ' ||
networks.slug || ' ' || networks.slug || ' ' ||
networks.url || ' ' ||
sites.name || ' ' || sites.name || ' ' ||
sites.slug || ' ' || sites.slug || ' ' ||
COALESCE(sites.url, '') || ' ' ||
COALESCE(sites.alias, '') || ' ' || COALESCE(sites.alias, '') || ' ' ||
COALESCE(releases.shoot_id, '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' || COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
@ -386,7 +388,7 @@ async function updateReleasesSearch(releaseIds) {
LEFT JOIN tags ON local_tags.tag_id = tags.id LEFT JOIN tags ON local_tags.tag_id = tags.id
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
GROUP BY releases.id, sites.name, sites.slug, sites.alias, networks.name, networks.slug; GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url;
`, releaseIds && [releaseIds]); `, releaseIds && [releaseIds]);
if (documents.rows?.length > 0) { if (documents.rows?.length > 0) {

View File

@ -1,10 +1,28 @@
'use strict'; 'use strict';
const config = require('config');
const tunnel = require('tunnel');
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const taskQueue = require('promise-task-queue'); const taskQueue = require('promise-task-queue');
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const proxyAgent = tunnel.httpsOverHttp({
proxy: {
host: config.proxy.host,
port: config.proxy.port,
},
});
function useProxy(url) {
if (!config.proxy.enable) {
return false;
}
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
}
const queue = taskQueue(); const queue = taskQueue();
queue.on('concurrencyReached:httpGet', () => { queue.on('concurrencyReached:httpGet', () => {
@ -15,47 +33,42 @@ queue.on('concurrencyReached:httpPost', () => {
logger.silly('Queueing POST requests'); logger.silly('Queueing POST requests');
}); });
queue.define('httpGet', async ({ queue.define('http', async ({
url,
timeout = 30000,
options = {},
}) => {
logger.silly(`GET ${url}`);
const res = await bhttp.get(url, {
responseTimeout: timeout,
...options,
});
res.code = res.statusCode;
return res;
}, {
concurrency: 20,
});
queue.define('httpPost', async ({
url, url,
method = 'GET',
body, body,
timeout = 30000, timeout = 30000,
options = {}, options = {},
}) => { }) => {
logger.silly(`POST ${url} with ${body}`); if (body) {
logger.silly(`${method.toUpperCase()} ${url} with ${body}`);
} else {
logger.silly(`${method.toUpperCase()} ${url}`);
}
const res = await bhttp.post(url, body, { const reqOptions = {
responseTimeout: timeout, responseTimeout: timeout,
...options, ...options,
}); };
res.code = res.statusCode; if (useProxy(url)) {
reqOptions.agent = proxyAgent;
}
return res; const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())
? await bhttp[method.toLowerCase()](url, body, reqOptions)
: await bhttp[method.toLowerCase()](url, reqOptions);
return {
...res,
code: res.statusCode,
};
}, { }, {
concurrency: 20, concurrency: 20,
}); });
async function get(url, options) { async function get(url, options) {
return queue.push('httpGet', { return queue.push('http', {
method: 'get', method: 'get',
url, url,
options, options,
@ -63,7 +76,7 @@ async function get(url, options) {
} }
async function post(url, body, options) { async function post(url, body, options) {
return queue.push('httpPost', { return queue.push('http', {
url, url,
body, body,
options, options,

View File

@ -287,7 +287,7 @@ function extractAll(htmlValue, selector) {
return initAll(window.document, selector, window); return initAll(window.document, selector, window);
} }
async function get(urlValue, selector, headers, queryAll = false) { async function get(urlValue, selector, headers, options, queryAll = false) {
const res = await http.get(urlValue, { const res = await http.get(urlValue, {
headers, headers,
}); });
@ -315,8 +315,8 @@ async function get(urlValue, selector, headers, queryAll = false) {
}; };
} }
async function getAll(urlValue, selector, headers) { async function getAll(urlValue, selector, headers, options) {
return get(urlValue, selector, headers, true); return get(urlValue, selector, headers, options, true);
} }
module.exports = { module.exports = {