DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Compare commits

base: DebaucheryLibrarian:c97d18fcf93e9ab4638dc2a1687cecf5fd6e191c
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental
..
compare: DebaucheryLibrarian:56a7fb0ad9bde1717e5387e33fc38a3f11557565
Branches Tags
DebaucheryLibrarian:master
DebaucheryLibrarian:old
DebaucheryLibrarian:experimental

No commits in common. "c97d18fcf93e9ab4638dc2a1687cecf5fd6e191c" and "56a7fb0ad9bde1717e5387e33fc38a3f11557565" have entirely different histories.
c97d18fcf9 ... 56a7fb0ad9

4 changed files with 6 additions and 14 deletions
Show Stats Expand all files Collapse all files

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.203.1", "version": "1.203.0",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "traxxx", "name": "traxxx",
"version": "1.203.1", "version": "1.203.0",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@casl/ability": "^5.2.2", "@casl/ability": "^5.2.2",

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "traxxx", "name": "traxxx",
"version": "1.203.1", "version": "1.203.0",
"description": "All the latest porn releases in one place", "description": "All the latest porn releases in one place",
"main": "src/app.js", "main": "src/app.js",
"scripts": { "scripts": {

7
src/deep.js
View File

@ -1,6 +1,5 @@
'use strict'; 'use strict';
const util = require('util');
const Promise = require('bluebird'); const Promise = require('bluebird');
const { mergeAdvanced: merge } = require('object-merge-advanced'); const { mergeAdvanced: merge } = require('object-merge-advanced');
@ -12,8 +11,6 @@ const qu = require('./utils/qu');
const getRecursiveParameters = require('./utils/get-recursive-parameters'); const getRecursiveParameters = require('./utils/get-recursive-parameters');
const windows = require('./utils/http-windows'); const windows = require('./utils/http-windows');
const waitImmediate = util.promisify(setImmediate);
function toBaseReleases(baseReleasesOrUrls, entity = null) { function toBaseReleases(baseReleasesOrUrls, entity = null) {
if (!baseReleasesOrUrls) { if (!baseReleasesOrUrls) {
return []; return [];
@ -131,8 +128,6 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
windows.get(pathname)?.close(); windows.get(pathname)?.close();
windows.delete(pathname); windows.delete(pathname);
await waitImmediate;
logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`); logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
const scrapedRelease = rawScrapedRelease?.scene || rawScrapedRelease; const scrapedRelease = rawScrapedRelease?.scene || rawScrapedRelease;
@ -201,7 +196,7 @@ async function scrapeReleases(baseReleases, entitiesBySlug, type) {
return Promise.map( return Promise.map(
baseReleases, baseReleases,
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type), async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
{ concurrency: 3 }, { concurrency: 5 },
); );
} }

7
src/utils/jsdom-perf.js
View File

@ -1,12 +1,9 @@
'use strict'; 'use strict';
const util = require('util');
const fs = require('fs').promises; const fs = require('fs').promises;
const Promise = require('bluebird'); const Promise = require('bluebird');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const waitImmediate = util.promisify(setImmediate);
async function init() { async function init() {
let peak = 0; let peak = 0;
const files = await fs.readdir('./html'); const files = await fs.readdir('./html');
@ -22,9 +19,9 @@ async function init() {
console.log(`Memory usage: ${usage.toFixed(2)} MB, peak ${peak.toFixed(2)} MB`); console.log(`Memory usage: ${usage.toFixed(2)} MB, peak ${peak.toFixed(2)} MB`);
await waitImmediate; await Promise.delay(100);
}, { }, {
concurrency: 100, concurrency: 10,
}); });
await Promise.delay(2000); await Promise.delay(2000);