forked from DebaucheryLibrarian/traxxx
Updated unprint, integrated throttle settings.
This commit is contained in:
@@ -33,8 +33,12 @@ unprint.options({
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
},
|
||||
limits: config.limits,
|
||||
});
|
||||
|
||||
unprint.on('requestInit', (event) => logger.debug(`Unprint ${event.method} (${event.interval}ms/${event.concurrency}p) ${event.url}`));
|
||||
unprint.on('requestError', (event) => logger.error(`Unprint to fetch ${event.url} (${event.status}): ${event.statusText}`));
|
||||
|
||||
/*
|
||||
function logActive() {
|
||||
setTimeout(() => {
|
||||
|
||||
@@ -93,6 +93,8 @@ async function fetchLatest(channel, page) {
|
||||
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
|
||||
const res = await unprint.get(`${channel.url}/latest/${page}`, { selectAll: '.card-scene' }); // studios as channels
|
||||
|
||||
console.log(res.ok, res.status, res.context.length, channel.url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user