DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Updated unprint, integrated throttle settings.

This commit is contained in:
DebaucheryLibrarian 2024-10-24 02:03:30 +02:00
parent 790e77fc1f
commit 4f247d23c4
6 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common

@ -1 +1 @@
Subproject commit e55818ab448d463c4765c3394a6049280799ec33 Subproject commit f69e343d8941b83e706c6af05e74be54e87d85a6

4
config/default.js
View File

@ -389,6 +389,10 @@ module.exports = {
interval: 50, interval: 50,
concurrency: 1, concurrency: 1,
}, },
'www.analvids.com': { // seems to error at default rate
interval: 500,
concurrency: 1,
},
}, },
fetchAfter: [1, 'week'], fetchAfter: [1, 'week'],
missingDateLimit: 3, missingDateLimit: 3,

8
package-lock.json generated
View File

@ -89,7 +89,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.11.13", "unprint": "^0.13.2",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",
@ -18312,9 +18312,9 @@
} }
}, },
"node_modules/unprint": { "node_modules/unprint": {
"version": "0.11.13", "version": "0.13.2",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.13.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz",
"integrity": "sha512-dEa3zdaXtK2TmRVWf4APunTUXZfnYb0Yv4RlddpFVA8fgYf0ER/m0JN/ZcbEfqg3x5YPiJEHpgLGH9pMv5lbqA==", "integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==",
"dependencies": { "dependencies": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",

2
package.json
View File

@ -148,7 +148,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.11.13", "unprint": "^0.13.2",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",

4
src/app.js
View File

@ -33,8 +33,12 @@ unprint.options({
headers: { headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}, },
limits: config.limits,
}); });
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint to fetch ${event.url} (${event.status}): ${event.statusText}`));
/* /*
function logActive() { function logActive() {
setTimeout(() => { setTimeout(() => {

2
src/scrapers/analvids.js
View File

@ -93,6 +93,8 @@ async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel // const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels
console.log(res.ok, res.status, res.context.length, channel.url);
if (res.ok) { if (res.ok) {
return scrapeAll(res.context, channel); return scrapeAll(res.context, channel);
} }