Accomodating unprint proxy options.
This commit is contained in:
parent
928650b602
commit
0fe6a1119a
2
common
2
common
|
@ -1 +1 @@
|
|||
Subproject commit 7a39529e2c20faa7153b8ffe932842e30b46aeb4
|
||||
Subproject commit 1122b4198f2c5bc27f9f4e3f3aedea6c81f09ba1
|
|
@ -301,7 +301,6 @@ module.exports = {
|
|||
},
|
||||
proxy: {
|
||||
enable: false,
|
||||
protocol: 'http',
|
||||
host: '',
|
||||
port: 8888,
|
||||
hostnames: [
|
||||
|
@ -335,6 +334,7 @@ module.exports = {
|
|||
'cdn.tushyraw.com',
|
||||
'cdn.slayed.com',
|
||||
'cdn.milfy.com',
|
||||
'www.analvids.com',
|
||||
'www.vogov.com',
|
||||
'www.vogov.com',
|
||||
'www.nubiles.net',
|
||||
|
|
|
@ -89,7 +89,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.14.1",
|
||||
"unprint": "^0.14.3",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
@ -18312,9 +18312,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/unprint": {
|
||||
"version": "0.14.1",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.14.1.tgz",
|
||||
"integrity": "sha512-LpsktR7NK3iDaYfy1HpNOiYoKGzLSq6wDhQN7RcwTQVJMz9kE0qQ8DS+ru2L76j52lq4v6oBktpnghbe//s3Mw==",
|
||||
"version": "0.14.3",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.14.3.tgz",
|
||||
"integrity": "sha512-frhe+/Cc9YzDhPmWB9oyTRWl6PPgtr91ayQkPx0J+5O1UlRd/cIcqF8RdhZnjl6DjibYB5aCX7+kmXGsQnqXLg==",
|
||||
"dependencies": {
|
||||
"axios": "^0.27.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
|
|
|
@ -148,7 +148,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.14.1",
|
||||
"unprint": "^0.14.3",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
|
|
@ -36,11 +36,11 @@ unprint.options({
|
|||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
},
|
||||
limits: config.limits,
|
||||
proxyAddress: `http://${config.proxy.host}:${config.proxy.port}`,
|
||||
proxy: config.proxy,
|
||||
});
|
||||
|
||||
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.proxy ? '' : ' proxied'}) ${event.url}`));
|
||||
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.proxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
|
||||
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}) ${event.url}`));
|
||||
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
|
||||
|
||||
function logActive() {
|
||||
setTimeout(() => {
|
||||
|
|
|
@ -91,9 +91,7 @@ function scrapeProfile({ query }, url, channel) {
|
|||
|
||||
async function fetchLatest(channel, page) {
|
||||
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
|
||||
const res = await unprint.get(`${channel.url}/latest/${page}`, {
|
||||
selectAll: '.card-scene', // studios as channels
|
||||
});
|
||||
const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
|
|
Loading…
Reference in New Issue