DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Accomodating unprint proxy options.

This commit is contained in:
DebaucheryLibrarian 2024-10-31 05:47:54 +01:00
parent 928650b602
commit 0fe6a1119a
6 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common

@ -1 +1 @@
Subproject commit 7a39529e2c20faa7153b8ffe932842e30b46aeb4 Subproject commit 1122b4198f2c5bc27f9f4e3f3aedea6c81f09ba1

2
config/default.js
View File

@ -301,7 +301,6 @@ module.exports = {
}, },
proxy: { proxy: {
enable: false, enable: false,
protocol: 'http',
host: '', host: '',
port: 8888, port: 8888,
hostnames: [ hostnames: [
@ -335,6 +334,7 @@ module.exports = {
'cdn.tushyraw.com', 'cdn.tushyraw.com',
'cdn.slayed.com', 'cdn.slayed.com',
'cdn.milfy.com', 'cdn.milfy.com',
'www.analvids.com',
'www.vogov.com', 'www.vogov.com',
'www.vogov.com', 'www.vogov.com',
'www.nubiles.net', 'www.nubiles.net',

8
package-lock.json generated
View File

@ -89,7 +89,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.14.1", "unprint": "^0.14.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",
@ -18312,9 +18312,9 @@
} }
}, },
"node_modules/unprint": { "node_modules/unprint": {
"version": "0.14.1", "version": "0.14.3",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.14.1.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.14.3.tgz",
"integrity": "sha512-LpsktR7NK3iDaYfy1HpNOiYoKGzLSq6wDhQN7RcwTQVJMz9kE0qQ8DS+ru2L76j52lq4v6oBktpnghbe//s3Mw==", "integrity": "sha512-frhe+/Cc9YzDhPmWB9oyTRWl6PPgtr91ayQkPx0J+5O1UlRd/cIcqF8RdhZnjl6DjibYB5aCX7+kmXGsQnqXLg==",
"dependencies": { "dependencies": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",

2
package.json
View File

@ -148,7 +148,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.14.1", "unprint": "^0.14.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",

6
src/app.js
View File

@ -36,11 +36,11 @@ unprint.options({
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}, },
limits: config.limits, limits: config.limits,
proxyAddress: `http://${config.proxy.host}:${config.proxy.port}`, proxy: config.proxy,
}); });
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.proxy ? '' : ' proxied'}) ${event.url}`)); unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p${event.isProxied ? ' proxied' : ''}) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.proxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`)); unprint.on('requestError', (event) => logger.error(`Unprint failed ${event.isProxied ? ' proxied' : ''}${event.method} ${event.url} (${event.status}): ${event.statusText}`));
function logActive() { function logActive() {
setTimeout(() => { setTimeout(() => {

4
src/scrapers/analvids.js
View File

@ -91,9 +91,7 @@ function scrapeProfile({ query }, url, channel) {
async function fetchLatest(channel, page) { async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel // const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
const res = await unprint.get(`${channel.url}/latest/${page}`, { const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels
selectAll: '.card-scene', // studios as channels
});
if (res.ok) { if (res.ok) {
return scrapeAll(res.context, channel); return scrapeAll(res.context, channel);