diff --git a/common b/common index e55818ab..f69e343d 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit e55818ab448d463c4765c3394a6049280799ec33 +Subproject commit f69e343d8941b83e706c6af05e74be54e87d85a6 diff --git a/config/default.js b/config/default.js index 8e8ea6bc..918e4a9b 100755 --- a/config/default.js +++ b/config/default.js @@ -389,6 +389,10 @@ module.exports = { interval: 50, concurrency: 1, }, + 'www.analvids.com': { // seems to error at default rate + interval: 500, + concurrency: 1, + }, }, fetchAfter: [1, 'week'], missingDateLimit: 3, diff --git a/package-lock.json b/package-lock.json index 5944068e..6e5069ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,7 +89,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.11.13", + "unprint": "^0.13.2", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", @@ -18312,9 +18312,9 @@ } }, "node_modules/unprint": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.11.13.tgz", - "integrity": "sha512-dEa3zdaXtK2TmRVWf4APunTUXZfnYb0Yv4RlddpFVA8fgYf0ER/m0JN/ZcbEfqg3x5YPiJEHpgLGH9pMv5lbqA==", + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz", + "integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index c417ab49..449627e0 100755 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.11.13", + "unprint": "^0.13.2", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", diff --git a/src/app.js b/src/app.js index 97032155..679db054 100755 --- a/src/app.js +++ b/src/app.js @@ -33,8 +33,12 @@ unprint.options({ headers: { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', }, + limits: config.limits, }); +unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p) ${event.url}`)); +unprint.on('requestError', (event) => logger.error(`Unprint to fetch ${event.url} (${event.status}): ${event.statusText}`)); + /* function logActive() { setTimeout(() => { diff --git a/src/scrapers/analvids.js b/src/scrapers/analvids.js index 0c386893..2e518d06 100644 --- a/src/scrapers/analvids.js +++ b/src/scrapers/analvids.js @@ -93,6 +93,8 @@ async function fetchLatest(channel, page) { // const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels + console.log(res.ok, res.status, res.context.length, channel.url); + if (res.ok) { return scrapeAll(res.context, channel); }