Refactored Bang! scraper to match new website, first use of unprint.

This commit is contained in:
DebaucheryLibrarian
2022-11-27 04:22:58 +01:00
parent 3cf8776ca5
commit 6edd62c337
61 changed files with 2371 additions and 1233 deletions

View File

@@ -2,6 +2,7 @@
const config = require('config');
const util = require('util');
const unprint = require('unprint');
// const log = require('why-is-node-running');
const Inspector = require('inspector-api');
const fs = require('fs').promises;
@@ -25,6 +26,13 @@ const getFileEntries = require('./utils/file-entries');
const inspector = new Inspector();
let done = false;
unprint.options({
timeout: 5000,
headers: {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
},
});
/*
function logActive() {
setTimeout(() => {