Updated unprint, integrated throttle settings.

This commit is contained in:
DebaucheryLibrarian
2024-10-24 02:03:30 +02:00
parent 790e77fc1f
commit 4f247d23c4
6 changed files with 16 additions and 6 deletions

View File

@@ -33,8 +33,12 @@ unprint.options({
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
},
limits: config.limits,
});
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p) ${event.url}`));
unprint.on('requestError', (event) => logger.error(`Unprint to fetch ${event.url} (${event.status}): ${event.statusText}`));
/*
function logActive() {
setTimeout(() => {

View File

@@ -93,6 +93,8 @@ async function fetchLatest(channel, page) {
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels
console.log(res.ok, res.status, res.context.length, channel.url);
if (res.ok) {
return scrapeAll(res.context, channel);
}