Compare commits
No commits in common. "9f37ec4cff6cffb4bcf857450d85f94a91459608" and "883e57ce1f1cf49612a563a913d1f2fa7bd77876" have entirely different histories.
9f37ec4cff
...
883e57ce1f
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "traxxx",
|
"name": "traxxx",
|
||||||
"version": "1.203.6",
|
"version": "1.203.5",
|
||||||
"lockfileVersion": 2,
|
"lockfileVersion": 2,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "traxxx",
|
"name": "traxxx",
|
||||||
"version": "1.203.6",
|
"version": "1.203.5",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@casl/ability": "^5.2.2",
|
"@casl/ability": "^5.2.2",
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "traxxx",
|
"name": "traxxx",
|
||||||
"version": "1.203.6",
|
"version": "1.203.5",
|
||||||
"description": "All the latest porn releases in one place",
|
"description": "All the latest porn releases in one place",
|
||||||
"main": "src/app.js",
|
"main": "src/app.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|
|
@ -130,11 +130,11 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
|
||||||
|
|
||||||
if (windows.has(pathname)) {
|
if (windows.has(pathname)) {
|
||||||
logger.debug(`Closing window for ${pathname}`);
|
logger.debug(`Closing window for ${pathname}`);
|
||||||
|
|
||||||
windows.get(pathname).close();
|
|
||||||
windows.delete(pathname);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
windows.get(pathname)?.close();
|
||||||
|
windows.delete(pathname);
|
||||||
|
|
||||||
await waitImmediate;
|
await waitImmediate;
|
||||||
|
|
||||||
logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
|
logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
|
||||||
|
|
|
@ -10,7 +10,7 @@ const tunnel = require('tunnel');
|
||||||
const Bottleneck = require('bottleneck');
|
const Bottleneck = require('bottleneck');
|
||||||
const { JSDOM, toughCookie } = require('jsdom');
|
const { JSDOM, toughCookie } = require('jsdom');
|
||||||
|
|
||||||
// const windows = require('./http-windows');
|
const windows = require('./http-windows');
|
||||||
|
|
||||||
const logger = require('../logger')(__filename);
|
const logger = require('../logger')(__filename);
|
||||||
const virtualConsole = require('./virtual-console')(__filename);
|
const virtualConsole = require('./virtual-console')(__filename);
|
||||||
|
@ -119,11 +119,9 @@ async function finalizeResult(res, options) {
|
||||||
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
|
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
|
||||||
const pathname = new URL(res.request.url).pathname.replace(/\//g, '_');
|
const pathname = new URL(res.request.url).pathname.replace(/\//g, '_');
|
||||||
|
|
||||||
/* unable to clear from update scrapes, nor any way to distinguish deep scrapes
|
|
||||||
if (window) {
|
if (window) {
|
||||||
windows.set(pathname, window);
|
windows.set(pathname, window);
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
|
|
||||||
if (argv.saveHtml) {
|
if (argv.saveHtml) {
|
||||||
await fs.writeFile(`./html/${pathname}.html`, html);
|
await fs.writeFile(`./html/${pathname}.html`, html);
|
||||||
|
|
Loading…
Reference in New Issue