diff --git a/package-lock.json b/package-lock.json index 861b3597..e01def5e 100755 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,8 @@ "promise-task-queue": "^1.2.0", "prop-types": "^15.7.2", "puppeteer": "^18.2.0", + "puppeteer-extra": "^3.3.4", + "puppeteer-extra-plugin-stealth": "^2.11.1", "sharp": "^0.29.2", "showdown": "^1.9.1", "source-map-support": "^0.5.16", @@ -76,7 +78,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.7.2", + "unprint": "^0.8.0", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", @@ -3201,6 +3203,14 @@ "node": ">= 10" } }, + "node_modules/@types/debug": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", + "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/eslint": { "version": "7.28.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.2.tgz", @@ -3252,6 +3262,11 @@ "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, + "node_modules/@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==" + }, "node_modules/@types/node": { "version": "16.11.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.6.tgz", @@ -10761,6 +10776,14 @@ "language-subtag-registry": "~0.3.2" } }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/lcid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", @@ -11311,6 +11334,97 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-deep/node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -11534,6 +11648,34 @@ "node": ">=0.10.0" } }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", @@ -13640,6 +13782,137 @@ } } }, + "node_modules/puppeteer-extra": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz", + "integrity": "sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w==", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "@types/puppeteer": "*", + "puppeteer": "*", + "puppeteer-core": "*" + }, + "peerDependenciesMeta": { + "@types/puppeteer": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "puppeteer-core": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz", + "integrity": "sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ==", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + }, + "engines": { + "node": ">=9.11.2" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-stealth": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz", + "integrity": "sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw==", + "dependencies": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.2", + "puppeteer-extra-plugin-user-preferences": "^2.4.0" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz", + "integrity": "sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA==", + "dependencies": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.2", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-preferences": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz", + "integrity": "sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A==", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.2", + "puppeteer-extra-plugin-user-data-dir": "^2.4.0" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -16735,9 +17008,9 @@ } }, "node_modules/unprint": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.7.2.tgz", - "integrity": "sha512-uG76G9b0NCpmggvtG1Nj1cgOVlQp46OEWfhIEi/VdDSIvPZJ5nSS8hqfA6ljR6V6/P8ZXVYDnV7PFJHh/4Imsg==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.8.0.tgz", + "integrity": "sha512-PBVdumsc7DfLwwTeSD5xSqITj3PFTUr/wA4unUykzov05YMp0a+Z3tVM1H7JFLZuRgzHRfKDbEpvAcj8vfYlLA==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", @@ -20608,6 +20881,14 @@ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" }, + "@types/debug": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", + "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==", + "requires": { + "@types/ms": "*" + } + }, "@types/eslint": { "version": "7.28.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.2.tgz", @@ -20659,6 +20940,11 @@ "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, + "@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==" + }, "@types/node": { "version": "16.11.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.6.tgz", @@ -26450,6 +26736,11 @@ "language-subtag-registry": "~0.3.2" } }, + "lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==" + }, "lcid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", @@ -26903,6 +27194,77 @@ } } }, + "merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "requires": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "dependencies": { + "clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "requires": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + } + }, + "for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "requires": { + "for-in": "^1.0.1" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + }, + "shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "requires": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "dependencies": { + "kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "requires": { + "is-buffer": "^1.0.2" + } + }, + "lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==" + } + } + } + } + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -27067,6 +27429,27 @@ "is-extendable": "^1.0.1" } }, + "mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "requires": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "dependencies": { + "for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==" + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + } + } + }, "mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", @@ -28658,6 +29041,58 @@ } } }, + "puppeteer-extra": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.4.tgz", + "integrity": "sha512-fN5pHvSMJ8d1o7Z8wLLTQOUBpORD2BcFn+KDs7QnkGZs9SV69hcUcce67vX4L4bNSEG3A0P6Osrv+vWNhhdm8w==", + "requires": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + } + }, + "puppeteer-extra-plugin": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.2.tgz", + "integrity": "sha512-0uatQxzuVn8yegbrEwSk03wvwpMB5jNs7uTTnermylLZzoT+1rmAQaJXwlS3+vADUbw6ELNgNEHC7Skm0RqHbQ==", + "requires": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + } + }, + "puppeteer-extra-plugin-stealth": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.1.tgz", + "integrity": "sha512-n0wdC0Ilc9tk5L6FWLyd0P2gT8b2fp+2NuB+KB0oTSw3wXaZ0D6WNakjJsayJ4waGzIJFCUHkmK9zgx5NKMoFw==", + "requires": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.2", + "puppeteer-extra-plugin-user-preferences": "^2.4.0" + } + }, + "puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.0.tgz", + "integrity": "sha512-qrhYPTGIqzL2hpeJ5DXjf8xMy5rt1UvcqSgpGTTOUOjIMz1ROWnKHjBoE9fNBJ4+ToRZbP8MzIDXWlEk/e1zJA==", + "requires": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.2", + "rimraf": "^3.0.2" + } + }, + "puppeteer-extra-plugin-user-preferences": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.0.tgz", + "integrity": "sha512-4XxMhMkJ+qqLsPY9ULF90qS9Bj1Qrwwgp1TY9zTdp1dJuy7QSgYE7xlyamq3cKrRuzg3QUOqygJo52sVeXSg5A==", + "requires": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.2", + "puppeteer-extra-plugin-user-data-dir": "^2.4.0" + } + }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -31027,9 +31462,9 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "unprint": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.7.2.tgz", - "integrity": "sha512-uG76G9b0NCpmggvtG1Nj1cgOVlQp46OEWfhIEi/VdDSIvPZJ5nSS8hqfA6ljR6V6/P8ZXVYDnV7PFJHh/4Imsg==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.8.0.tgz", + "integrity": "sha512-PBVdumsc7DfLwwTeSD5xSqITj3PFTUr/wA4unUykzov05YMp0a+Z3tVM1H7JFLZuRgzHRfKDbEpvAcj8vfYlLA==", "requires": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index 7672e534..e4017a7e 100755 --- a/package.json +++ b/package.json @@ -126,6 +126,8 @@ "promise-task-queue": "^1.2.0", "prop-types": "^15.7.2", "puppeteer": "^18.2.0", + "puppeteer-extra": "^3.3.4", + "puppeteer-extra-plugin-stealth": "^2.11.1", "sharp": "^0.29.2", "showdown": "^1.9.1", "source-map-support": "^0.5.16", @@ -135,7 +137,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.7.2", + "unprint": "^0.8.0", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", diff --git a/public/img/locations/rocky_emerson_house/Fkb-OiHVEAIg2zL.jpg b/public/img/locations/rocky_emerson_house/Fkb-OiHVEAIg2zL.jpg new file mode 100644 index 00000000..64d5ced1 Binary files /dev/null and b/public/img/locations/rocky_emerson_house/Fkb-OiHVEAIg2zL.jpg differ diff --git a/public/img/locations/rocky_emerson_house/Fkb-OiJVQAIVx1O.jpg b/public/img/locations/rocky_emerson_house/Fkb-OiJVQAIVx1O.jpg new file mode 100644 index 00000000..4729f93f Binary files /dev/null and b/public/img/locations/rocky_emerson_house/Fkb-OiJVQAIVx1O.jpg differ diff --git a/public/img/locations/rocky_emerson_house/Fkb-OiNUYAAdpqr.jpg b/public/img/locations/rocky_emerson_house/Fkb-OiNUYAAdpqr.jpg new file mode 100644 index 00000000..bf3d1b78 Binary files /dev/null and b/public/img/locations/rocky_emerson_house/Fkb-OiNUYAAdpqr.jpg differ diff --git a/public/img/locations/rocky_emerson_house/email b/public/img/locations/rocky_emerson_house/email new file mode 100644 index 00000000..7830d4fa --- /dev/null +++ b/public/img/locations/rocky_emerson_house/email @@ -0,0 +1 @@ +E-mail Rocky Emerson for permission to use these photos https://twitter.com/RockyEmersonXXX/status/1605387352087547905 diff --git a/public/img/logos/suddenimpact/misc/sudden-impact.png b/public/img/logos/suddenimpact/misc/sudden-impact.png new file mode 100644 index 00000000..cf229866 Binary files /dev/null and b/public/img/logos/suddenimpact/misc/sudden-impact.png differ diff --git a/public/img/logos/suddenimpact/misc/sudden-impact_light.png b/public/img/logos/suddenimpact/misc/sudden-impact_light.png new file mode 100644 index 00000000..ddbda1dd Binary files /dev/null and b/public/img/logos/suddenimpact/misc/sudden-impact_light.png differ diff --git a/public/img/logos/suddenimpact/suddenimpact.png b/public/img/logos/suddenimpact/suddenimpact.png new file mode 100644 index 00000000..80b1dfef Binary files /dev/null and b/public/img/logos/suddenimpact/suddenimpact.png differ diff --git a/src/app.js b/src/app.js index 374ed1bf..a5d3bd4c 100755 --- a/src/app.js +++ b/src/app.js @@ -208,6 +208,7 @@ async function init() { } await http.destroyBypassSessions(); + await http.destroyBrowserSessions(); knex.destroy(); done = true; diff --git a/src/deep.js b/src/deep.js index ee3956a1..9ec4b0ff 100755 --- a/src/deep.js +++ b/src/deep.js @@ -221,10 +221,14 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') { async function scrapeReleases(baseReleases, entitiesBySlug, type) { const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesBySlug).map(async ([slug, entity]) => { + console.log('scraper', entity.scraper?.beforeFetchScenes); + if (entity.scraper?.beforeFetchScenes) { const parameters = getRecursiveParameters(entity); const preData = await entity.scraper.beforeFetchScenes(entity, parameters); + console.log('pre data', preData); + return [slug, { ...entity, preData }]; } diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js index aac1b637..c6fb1759 100755 --- a/src/scrapers/kink.js +++ b/src/scrapers/kink.js @@ -1,26 +1,34 @@ 'use strict'; +const unprint = require('unprint'); + const qu = require('../utils/qu'); const http = require('../utils/http'); +const slugify = require('../utils/slugify'); function scrapeAll(scenes) { return scenes.map(({ query }) => { const release = {}; - const href = query.url('.shoot-thumb-info > a'); - release.url = `https://kink.com${href}`; + const href = query.url('.shoot-link'); + + release.url = `https://www.kink.com${href}`; release.shootId = href.split('/').slice(-1)[0]; release.entryId = release.shootId; - release.title = query.q('.shoot-thumb-title a', true); + release.title = query.content('.shoot-thumb-title a', true); release.date = query.date('.date', 'MMM DD, YYYY'); - release.actors = query.all('.shoot-thumb-models a', true); - release.stars = query.q('.average-rating', 'data-rating') / 10; + release.actors = query.all('.shoot-thumb-models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null, { origin: 'https://www.kink.com' }), + })); + + release.rating = query.number('.thumb-ratings') / 10; release.poster = query.img('.adimage'); - release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map((photo) => [ + release.photos = query.imgs('.rollover .roll-image', { attribute: 'data-imagesrc' }).map((photo) => [ photo.replace('410/', '830/'), photo, ]); @@ -31,25 +39,28 @@ function scrapeAll(scenes) { }); } -async function scrapeScene({ query }, url) { +function scrapeScene({ query }, url) { const release = { url }; release.shootId = new URL(url).pathname.split('/')[2]; release.entryId = release.shootId; - release.title = query.q('.shoot-title span.favorite-button', 'data-title'); - release.description = query.q('.description-text', true); + release.title = query.attribute('.shoot-title .favorite-button', 'data-title') || query.content('.shoot-title'); + release.description = query.content('.description-text'); release.date = query.date('.shoot-date', 'MMMM DD, YYYY'); - release.actors = query.all('.names a', true).map((actor) => actor.replace(/,\s*/, '')); - release.director = query.q('.director-name', true); + release.actors = query.elements('.names a').map((actorEl) => ({ + name: unprint.query.content(actorEl).replace(/,\s*/, ''), + url: unprint.query.url(actorEl, null, { origin: 'https://www.kink.com' }), + })); + release.director = query.content('.director-name'); release.photos = query.imgs('.gallery .thumb img, #gallerySlider .gallery-img', 'data-image-file'); release.poster = query.poster(); - release.tags = query.all('.tag-list a[href*="/tag"]', true).map((tag) => tag.replace(/,\s*/, '')); + release.tags = query.contents('.tag-list a[href*="/tag"]').map((tag) => tag.replace(/,\s*/, '')); - const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url'); + const trailer = query.attribute('.player span[data-type="trailer-src"]', 'data-url'); if (trailer) { release.trailer = [ @@ -72,7 +83,9 @@ async function scrapeScene({ query }, url) { ]; } - release.channel = query.url('.shoot-logo a').split('/').slice(-1)[0]; + release.channel = slugify(query.url('.shoot-logo a')?.split('/').slice(-1)[0], ''); + + console.log(release); return release; } @@ -131,33 +144,43 @@ async function scrapeProfile({ query }, actorUrl, include) { return profile; } -async function beforeFetchLatest() { - const tab = await http.getBrowserSession(); - - await tab.goto('https://www.kink.com'); - await tab.click('#ccc-recommended-settings', { delay: 1000 }); - await tab.click('#contentTypeModal button', { delay: 1000 }); - - return tab; -} - async function fetchLatest(site, page = 1) { - // const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', { - // const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', { - const res = await qu.getAll(`https://www.kink.com/search?type=shoots&channelIds=${site.slug}&sort=published&page=${page}`, '.shoot-list .shoot', { - cookie: 'ct=2;', - }, { - includeDefaultHeaders: false, - followRedirects: false, - }); + const { tab } = await http.getBrowserSession('kink', { headless: false }); + const res = await tab.goto(`https://www.kink.com/search?type=shoots&channelIds=${site.slug}&sort=published&page=${page}`); + const status = res.status(); - console.log(res.items); + if (status === 200) { + const html = await tab.content(); + const items = unprint.initAll(html, '.results .shoot-card'); - if (res.ok) { - return scrapeAll(res.items, site); + const scenes = scrapeAll(items, site); + + await tab.close(); + + return scenes; } - return res.status; + return status; +} + +async function fetchScene(url, channel) { + const { tab } = await http.getBrowserSession('kink'); + const res = await tab.goto(url); + + const status = res.status(); + + if (status === 200) { + const html = await tab.content(); + const item = unprint.init(html); + + const scene = scrapeScene(item, url, channel); + + await tab.close(); + + return scene; + } + + return status; } async function fetchProfile({ name: actorName }, entity, include) { @@ -185,8 +208,8 @@ async function fetchProfile({ name: actorName }, entity, include) { } module.exports = { - beforeFetchLatest, + // beforeNetwork, fetchLatest, + fetchScene, fetchProfile, - scrapeScene, }; diff --git a/src/updates.js b/src/updates.js index 7540b47f..45654a21 100755 --- a/src/updates.js +++ b/src/updates.js @@ -235,7 +235,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases, beforeNetwork) { } try { - const beforeFetchLatest = await scraper.beforeFetchLatest?.(channelEntity); + const beforeFetchLatest = await scraper.beforeFetchLatest?.(channelEntity, { beforeNetwork }); return await scrapeChannelReleases(layoutScraper, channelEntity, { ...accNetworkReleases, diff --git a/src/utils/http.js b/src/utils/http.js index 70fdb3b1..006a3b2b 100755 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -9,7 +9,8 @@ const stream = require('stream'); const tunnel = require('tunnel'); const Bottleneck = require('bottleneck'); const { JSDOM, toughCookie } = require('jsdom'); -const puppeteer = require('puppeteer'); +const puppeteer = require('puppeteer-extra'); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); const windows = require('./http-windows'); @@ -30,12 +31,13 @@ const limiters = { const bypassSessions = new Map(); let browser = null; -const browserSessions = new Map(); Promise.config({ cancellation: true, }); +puppeteer.use(StealthPlugin()); + const defaultOptions = { timeout: argv.requestTimeout, encodeJSON: true, @@ -156,26 +158,27 @@ function extractJson(solution) { return solution.response; } -async function getBrowserSession(hostname) { - console.log(browserSessions); +async function getBrowserSession(identifier, options = {}) { + return limiters.bypass.schedule(async () => { + if (!browser) { + browser = await puppeteer.launch({ + headless: typeof options.headless === 'undefined' ? true : options.headless, + // headless: false, + }); - if (browserSessions.has(hostname)) { - return browserSessions.get(hostname); - } + logger.info('Initialized puppeteer browser'); + } - if (!browser) { - browser = await puppeteer.launch({ headless: false }); - } + const tab = await browser.newPage(); - const page = await browser.newPage(); + logger.verbose(`Opened puppeteer tab${identifier ? `for ${identifier}` : ''}`); - browserSessions.set(hostname, page); - - return page; + return { browser, tab }; + }); } async function bypassBrowserRequest(url, _options) { - const page = await limiters.bypass.schedule(async () => getBrowserSession(new URL(url).hostname)); + const page = await getBrowserSession(new URL(url).hostname); const res = await page.goto(url); const body = await page.content(); @@ -254,6 +257,10 @@ async function destroyBypassSessions() { await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 }); } +async function destroyBrowserSessions() { + await browser?.close(); +} + async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) { const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass)); @@ -456,9 +463,11 @@ module.exports = { patch, session: getSession, cookieJar: getCookieJar, + browser, getBrowserSession, getBypassSession, getSession, getCookieJar, destroyBypassSessions, + destroyBrowserSessions, };