From b7dc3f5fd14e607c53749678e6db1af23dff9b13 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:58 +0200 Subject: [PATCH] Storing pHash. Fixed RedGIFs user agent. --- package-lock.json | 768 ++++++++++++++++++++++++++++++++++-- package.json | 1 + src/app.js | 4 +- src/curate/hashPost.js | 10 +- src/curate/posts.js | 12 +- src/fetch/content.js | 39 +- src/fetch/info.js | 12 +- src/fetch/item.js | 23 +- src/methods/imgurImage.js | 2 +- src/methods/redgifs.js | 5 +- src/save/mux.js | 33 +- src/save/profileDetails.js | 2 +- src/save/writeToIndex.js | 19 +- src/sources/getPosts.js | 6 +- src/sources/getUserPosts.js | 10 +- 15 files changed, 852 insertions(+), 94 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4e57e32..f9e6fc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "node-fetch": "^2.1.2", "object.omit": "^3.0.0", "object.pick": "^1.3.0", + "sharp-phash": "^2.1.0", "snoowrap": "^1.20.0", "template-format": "^1.2.4", "unprint": "^0.8.1", @@ -1064,6 +1065,26 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peer": true + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -1119,6 +1140,31 @@ "rhino" ] }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "peer": true, + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "peer": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/blake2": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/blake2/-/blake2-4.1.1.tgz", @@ -1189,6 +1235,30 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peer": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -1279,6 +1349,12 @@ "node": ">= 0.6" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "peer": true + }, "node_modules/cliui": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz", @@ -1592,6 +1668,15 @@ "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", "peer": true }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "peer": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -1628,6 +1713,15 @@ "node": ">=0.4.0" } }, + "node_modules/detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/dev-null": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dev-null/-/dev-null-0.1.1.tgz", @@ -2570,6 +2664,15 @@ "integrity": "sha512-IgCT0YmeZAzQK9OVZAXBUIZQ1HU2Ml12mwqyzKKyATqVxUwUghnqS7+gSUzmbfsJsQ1k6uV1OboDPGn7Y4bi5A==", "optional": true }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -2779,6 +2882,12 @@ "resolved": "https://registry.npmjs.org/form-fix-array/-/form-fix-array-1.0.0.tgz", "integrity": "sha512-f3qXI4CcvW7/6vqTKwCftcrFgfEBfWYPQTfvXrHYevHbJVfc107/SVvXvwUAYMaUAHdvu9ENQvLufJKphQI14w==" }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "peer": true + }, "node_modules/fs-extra": { "version": "5.0.0", "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==", @@ -2882,6 +2991,12 @@ "assert-plus": "^1.0.0" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "peer": true + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -3137,6 +3252,26 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peer": true + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -3178,9 +3313,15 @@ } }, "node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "peer": true }, "node_modules/internal-slot": { "version": "1.0.5", @@ -4108,6 +4249,12 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "peer": true + }, "node_modules/moment": { "version": "2.29.4", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", @@ -4137,11 +4284,68 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==" }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "peer": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" }, + "node_modules/node-abi": { + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.33.0.tgz", + "integrity": "sha512-7GGVawqyHF4pfd0YFybhv/eM9JwTtPqx0mAanQ146O3FlSh3pA24zf9IRQTOsfTSqXTNzPSP5iagAJ94jjuVog==", + "peer": true, + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "peer": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "peer": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "peer": true + }, + "node_modules/node-addon-api": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", + "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==", + "peer": true + }, "node_modules/node-cron": { "version": "1.2.1", "integrity": "sha512-lgci/ub6KWL6SUnKOIiMkhfjmUk3jvEuO/Ypa2/CGSXiC8z4x9EkwMx7Dcu7Dt4LktcfOl8/WcLT2x7gInBa7g==" @@ -4521,6 +4725,41 @@ "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "peer": true, + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -4563,6 +4802,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -4600,6 +4848,39 @@ } ] }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "peer": true, + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", @@ -4913,6 +5194,104 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "node_modules/sharp": { + "version": "0.31.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz", + "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==", + "hasInstallScript": true, + "peer": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.1", + "node-addon-api": "^5.0.0", + "prebuild-install": "^7.1.1", + "semver": "^7.3.8", + "simple-get": "^4.0.1", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp-phash": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sharp-phash/-/sharp-phash-2.1.0.tgz", + "integrity": "sha512-9JYWr4tiKpjRA5Mi0qHn6LP2evS+GjdRVGjDFOSnO761m5Pavkpm83SyzauO2Ntt7znVqTn7J3XTUwHjRPAonw==", + "engines": { + "node": ">= 10" + }, + "peerDependencies": { + "sharp": ">= 0.25.4" + } + }, + "node_modules/sharp/node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "peer": true, + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/sharp/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "peer": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/sharp/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "peer": true + }, + "node_modules/sharp/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "peer": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "peer": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "peer": true + }, "node_modules/shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -5246,6 +5625,48 @@ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "peer": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "peer": true, + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "peer": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/template-format": { "version": "1.2.4", "integrity": "sha512-+8ItNMtMTBbsEHyPR1l7Ke1WZfl91PAcoTvwAcx5U28CRLd7ylzDLazv0kuDTzNmdq/RAOnsxFVWzr4QwVIFVg==" @@ -6582,15 +7003,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/youtube-dl/node_modules/get-stream/node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "node_modules/youtube-dl/node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -7520,6 +7932,12 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "peer": true + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -7573,6 +7991,30 @@ } } }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "peer": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "peer": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "blake2": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/blake2/-/blake2-4.1.1.tgz", @@ -7623,6 +8065,16 @@ "update-browserslist-db": "^1.0.10" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "peer": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -7685,6 +8137,12 @@ "parse5": "^3.0.1" } }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "peer": true + }, "cliui": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz", @@ -7959,6 +8417,12 @@ } } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "peer": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -7983,6 +8447,12 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "peer": true + }, "dev-null": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dev-null/-/dev-null-0.1.1.tgz", @@ -8707,6 +9177,12 @@ "integrity": "sha512-IgCT0YmeZAzQK9OVZAXBUIZQ1HU2Ml12mwqyzKKyATqVxUwUghnqS7+gSUzmbfsJsQ1k6uV1OboDPGn7Y4bi5A==", "optional": true }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "peer": true + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -8872,6 +9348,12 @@ "resolved": "https://registry.npmjs.org/form-fix-array/-/form-fix-array-1.0.0.tgz", "integrity": "sha512-f3qXI4CcvW7/6vqTKwCftcrFgfEBfWYPQTfvXrHYevHbJVfc107/SVvXvwUAYMaUAHdvu9ENQvLufJKphQI14w==" }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "peer": true + }, "fs-extra": { "version": "5.0.0", "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==", @@ -8951,6 +9433,12 @@ "assert-plus": "^1.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "peer": true + }, "glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -9129,6 +9617,12 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "peer": true + }, "ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -9158,9 +9652,15 @@ } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "peer": true }, "internal-slot": { "version": "1.0.5", @@ -9850,6 +10350,12 @@ "minimist": "0.0.8" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "peer": true + }, "moment": { "version": "2.29.4", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", @@ -9873,11 +10379,58 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==" }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "peer": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" }, + "node-abi": { + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.33.0.tgz", + "integrity": "sha512-7GGVawqyHF4pfd0YFybhv/eM9JwTtPqx0mAanQ146O3FlSh3pA24zf9IRQTOsfTSqXTNzPSP5iagAJ94jjuVog==", + "peer": true, + "requires": { + "semver": "^7.3.5" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "peer": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "peer": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "peer": true + } + } + }, + "node-addon-api": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", + "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==", + "peer": true + }, "node-cron": { "version": "1.2.1", "integrity": "sha512-lgci/ub6KWL6SUnKOIiMkhfjmUk3jvEuO/Ypa2/CGSXiC8z4x9EkwMx7Dcu7Dt4LktcfOl8/WcLT2x7gInBa7g==" @@ -10158,6 +10711,34 @@ "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "peer": true, + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "peer": true + } + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -10197,6 +10778,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -10217,6 +10807,32 @@ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "peer": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "peer": true + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "peer": true + } + } + }, "react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", @@ -10436,6 +11052,79 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "sharp": { + "version": "0.31.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz", + "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==", + "peer": true, + "requires": { + "color": "^4.2.3", + "detect-libc": "^2.0.1", + "node-addon-api": "^5.0.0", + "prebuild-install": "^7.1.1", + "semver": "^7.3.8", + "simple-get": "^4.0.1", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "peer": true, + "requires": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "peer": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "peer": true + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "peer": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "peer": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "peer": true + } + } + }, + "sharp-phash": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sharp-phash/-/sharp-phash-2.1.0.tgz", + "integrity": "sha512-9JYWr4tiKpjRA5Mi0qHn6LP2evS+GjdRVGjDFOSnO761m5Pavkpm83SyzauO2Ntt7znVqTn7J3XTUwHjRPAonw==", + "requires": {} + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -10674,6 +11363,44 @@ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "peer": true, + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "peer": true, + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "peer": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "template-format": { "version": "1.2.4", "integrity": "sha512-+8ItNMtMTBbsEHyPR1l7Ke1WZfl91PAcoTvwAcx5U28CRLd7ylzDLazv0kuDTzNmdq/RAOnsxFVWzr4QwVIFVg==" @@ -11607,17 +12334,6 @@ "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", "requires": { "pump": "^3.0.0" - }, - "dependencies": { - "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } } }, "is-stream": { diff --git a/package.json b/package.json index c4ada91..bb9a932 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "node-fetch": "^2.1.2", "object.omit": "^3.0.0", "object.pick": "^1.3.0", + "sharp-phash": "^2.1.0", "snoowrap": "^1.20.0", "template-format": "^1.2.4", "unprint": "^0.8.1", diff --git a/src/app.js b/src/app.js index 6e4a075..aebcad4 100644 --- a/src/app.js +++ b/src/app.js @@ -50,6 +50,8 @@ async function fetchPredata(hosts) { if (methods[host?.method]?.fetchPredata) { const data = await methods[host.method].fetchPredata(); + logger.info(`Fetched predata for ${host.method}`); + return { ...acc, [host.method]: data, @@ -108,8 +110,6 @@ async function getDirectContent(links, ep) { const predata = await fetchPredata(hosts.map(({ host }) => host)); - console.log('app predata', predata); - return Promise.map(hosts, async ({ link, host }) => { const info = await getInfo(host, { reddit, link, predata }); diff --git a/src/curate/hashPost.js b/src/curate/hashPost.js index 88d1676..b9eeb79 100644 --- a/src/curate/hashPost.js +++ b/src/curate/hashPost.js @@ -2,9 +2,11 @@ const crypto = require('crypto'); -const hashPost = post => crypto - .createHash('md5') - .update(post.id + post.subreddit_id + post.created_utc + post.title) - .digest('hex'); +function hashPost(post) { + return crypto + .createHash('md5') + .update(post.id + post.subreddit_id + post.created_utc + post.title) + .digest('hex'); +} module.exports = hashPost; diff --git a/src/curate/posts.js b/src/curate/posts.js index 79d0714..0d2117e 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -1,12 +1,12 @@ 'use strict'; const config = require('config'); +const { isAfter, isBefore, isEqual } = require('date-fns'); const omit = require('object.omit'); const dissectLink = require('../dissectLink'); const hashPost = require('./hashPost'); -const { isAfter, isBefore, isEqual } = require('date-fns'); const logger = require('../logger')(__filename); function report(curatedPosts, indexed, user, args) { @@ -58,7 +58,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) datetime: new Date(post.created_utc * 1000), subreddit: post.subreddit.display_name, score: post.score, - preview: post.preview ? post.preview.images.map(image => image.source) : null, + preview: post.preview ? post.preview.images.map((image) => image.source) : null, host, direct: post.direct, comments: post.comments, @@ -66,7 +66,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) }; if (indexed.entries.length) { - const indexedPost = indexed.entries.find(entry => entry.id === post.id); + const indexedPost = indexed.entries.find((entry) => entry.id === post.id); if (indexedPost && !args.redownload) { curatedPost.previewFallback = indexedPost.preview; @@ -96,7 +96,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) return acc; } - const ignoring = args.ignore ? args.ignore.find(prop => post[prop]) : null; + const ignoring = args.ignore ? args.ignore.find((prop) => post[prop]) : null; if (ignoring) { logger.verbose(`Ignoring ${ignoring} post '${post.title}' (${permalink})`); @@ -139,7 +139,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts).reduce((accPosts, user) => { const processed = new Set(); - const ignoreIds = new Set(ignoreIdsArray.map(postId => String(postId).toLowerCase())); + const ignoreIds = new Set(ignoreIdsArray.map((postId) => String(postId).toLowerCase())); const indexedByDate = user.indexed.original.sort((entryA, entryB) => new Date(entryA.date) - new Date(entryB.date)); const indexed = { @@ -162,7 +162,7 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts report(curatedPosts, indexed, user, args); - const indexedOriginal = user.indexed.original.filter(entry => !curatedPosts.indexedUpdated.find(post => post.id === entry.id)); + const indexedOriginal = user.indexed.original.filter((entry) => !curatedPosts.indexedUpdated.find((post) => post.id === entry.id)); return { ...accPosts, diff --git a/src/fetch/content.js b/src/fetch/content.js index 9a8f9c9..a1ec16d 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -58,7 +58,10 @@ async function getBuffers(item, context) { } const sources = item.mux ? [item.url].concat(item.mux) : [item.url]; - const buffers = await Promise.map(sources, (source) => fetchItem(source, 0, context)); + const buffers = await Promise.map(sources, (source) => fetchItem(source, 0, { + item, + ...context, + })); if (buffers.filter((buffer) => buffer).length > 0) { return buffers; @@ -105,31 +108,37 @@ function getFilepath(item, content, host, post, user) { async function fetchSaveUserContent(user, ep, args) { const profilePaths = await saveProfileDetails(user, args); - const hashes = new Set(user.indexed.original.map((item) => item.hash)); + const hashes = new Set(user.indexed.original.flatMap((item) => [item.hash, item.phash]).filter(Boolean)); const posts = await Promise.map(user.posts, async (post) => { if (!post.content) { return null; } - const hash = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => { + const items = await Promise.map(post.content.items, async (originalItem, index) => { const item = { ...originalItem, index }; const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers }); // no buffers, ignore item if (!buffers || buffers.length === 0) { - return accItems; + return null; } + item.hash = buffers[0].hash; + item.phash = buffers[0].phash; + // prevent duplicates - if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash)) { + if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash) && !item.album) { logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`); - return buffers[0].hash; + return { + ...item, + ignored: true, + }; } const filepath = getFilepath(item, post.content, post.host, post, user); - const sourcePaths = await save(filepath, buffers, item, post); + const sourcePaths = await save(filepath, buffers.map(({ buffer }) => buffer), item, post); hashes.add(buffers[0].hash); @@ -139,12 +148,14 @@ async function fetchSaveUserContent(user, ep, args) { await addMeta(filepath, item, post, user, ep); - return buffers[0].hash; - }, []); + return item; + }); return { ...post, - hash, + hash: items[0]?.hash, + phash: items[0]?.phash, + content: items.filter(Boolean), }; }, { concurrency: config.fetch.concurrency, @@ -154,7 +165,7 @@ async function fetchSaveUserContent(user, ep, args) { } async function fetchSaveDirectContent(content, host, ep) { - return Promise.reduce(content.items, async (accItems, originalItem, index) => { + return Promise.map(content.items, async (originalItem, index) => { logger.info(`Fetching and saving '${host.url}'`); const item = { ...originalItem, index }; @@ -162,7 +173,7 @@ async function fetchSaveDirectContent(content, host, ep) { // no buffers, ignore item if (!buffers || buffers.length === 0) { - return accItems; + return; } const filepath = getFilepath(item, content, host, null, null); @@ -173,9 +184,7 @@ async function fetchSaveDirectContent(content, host, ep) { } await addMeta(filepath, item, null, null, ep); - - return sourcePaths; - }, []); + }, { concurrency: 5 }); } module.exports = { diff --git a/src/fetch/info.js b/src/fetch/info.js index 57da353..8fdfa08 100644 --- a/src/fetch/info.js +++ b/src/fetch/info.js @@ -18,8 +18,6 @@ async function attachContentInfo(users, { reddit, predata }) { return accPosts; } - console.log('attach predata', predata[post.host.method]); - try { return [ ...accPosts, @@ -60,7 +58,10 @@ async function attachContentInfo(users, { reddit, predata }) { async function getInfo(host, { reddit, url, predata }) { if (host === null) { try { - const info = await methods.tube(host, null, reddit); + const info = await methods.tube(host, null, { + reddit, + predata: predata.tube, + }); return info; } catch (error) { @@ -70,7 +71,10 @@ async function getInfo(host, { reddit, url, predata }) { } } - return (methods[host.method].fetchInfo || methods[host.method])(host, null, { reddit, predata }); + return (methods[host.method].fetchInfo || methods[host.method])(host, null, { + reddit, + predata: predata[host.method], + }); } module.exports = { diff --git a/src/fetch/item.js b/src/fetch/item.js index 935055c..820fbe4 100644 --- a/src/fetch/item.js +++ b/src/fetch/item.js @@ -3,23 +3,26 @@ const config = require('config'); const bhttp = require('bhttp'); const blake2 = require('blake2'); +const phash = require('sharp-phash'); +// const phashDistance = require('sharp-phash/distance'); const logger = require('../logger')(__filename); const limiter = require('../limiter').items; -async function fetchItem(url, attempt, { post, host, headers }) { +async function fetchItem(url, attempt, context) { async function retry(error) { - logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`); + logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${context.post ? context.post.permalink : 'no post'})`); if (attempt < config.fetch.retries) { - return fetchItem(url, attempt + 1, { post, host, headers }); + return fetchItem(url, attempt + 1, context); } return null; } try { - const res = await limiter.schedule(async () => bhttp.get(url, { headers })); + // throw new Error('Failed it!'); + const res = await limiter.schedule(async () => bhttp.get(url, { headers: context.headers })); if (res.statusCode !== 200) { throw new Error(`Response not OK for ${url} (${res.statusCode}): ${res.body.toString()}`); @@ -29,13 +32,21 @@ async function fetchItem(url, attempt, { post, host, headers }) { throw new Error(`Unexpected response for ${url} (${res.statusCode}): ${res.body}`); } - logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`); + logger.debug(`Fetched '${context.host?.url || url}' (${context.post?.permalink || 'no post'})`); const hash = blake2.createHash('blake2b', { digestLength: 24 }); hash.update(res.body); const contentHash = hash.digest('hex'); - return Object.assign(res.body, { hash: contentHash }); + const phashResult = context.item?.type?.includes('image/') + ? await phash(res.body) + : null; + + return { + buffer: res.body, + hash: contentHash, + phash: phashResult, + }; } catch (error) { return retry(error); } diff --git a/src/methods/imgurImage.js b/src/methods/imgurImage.js index 8cb2c2f..242006b 100644 --- a/src/methods/imgurImage.js +++ b/src/methods/imgurImage.js @@ -31,7 +31,7 @@ async function fetchPredata() { return data; } -async function imgurImageApi(host, post, { predata } = {}) { +async function imgurImageApi(host, post, { predata }) { if (predata.remaining === 10) { // keep a buffer throw new Error(`Reached Imgur API rate limit with source '${host.url}'`); } diff --git a/src/methods/redgifs.js b/src/methods/redgifs.js index 8cf80eb..8fc6cf3 100644 --- a/src/methods/redgifs.js +++ b/src/methods/redgifs.js @@ -21,7 +21,6 @@ async function fetchPredata() { address: data.addr, agent: data.agent, token: data.token, - userAgent, }; } @@ -66,7 +65,7 @@ async function redgifsApi(host, post, { predata }) { const res = await fetch(`https://api.redgifs.com/v2/gifs/${host.id.toLowerCase()}`, { headers: { authorization: `Bearer ${predata.token}`, - 'user-agent': predata.userAgent, + 'user-agent': predata.agent, }, }); @@ -99,7 +98,7 @@ async function redgifsApi(host, post, { predata }) { original: data.gif, }], headers: { - 'user-agent': predata.userAgent, + 'user-agent': predata.agent, }, }; diff --git a/src/save/mux.js b/src/save/mux.js index 6e8dd4c..5f1bb27 100644 --- a/src/save/mux.js +++ b/src/save/mux.js @@ -5,23 +5,24 @@ const fs = require('fs-extra'); const logger = require('../logger')(__filename); -function mux(target, sources) { - return new Promise((resolve, reject) => sources.reduce((acc, source) => acc.input(source), ffmpeg()) - .videoCodec('copy') - .audioCodec('copy') - .on('start', () => { - logger.verbose(`Muxing ${sources.length} streams to '${target}'`); - }) - .on('end', (stdout) => { - logger.verbose(`Muxed and saved '${target}'`); +async function mux(target, sources) { + return new Promise((resolve, reject) => { + sources.reduce((acc, source) => acc.input(source), ffmpeg()) + .videoCodec('copy') + .audioCodec('copy') + .on('start', () => { + logger.verbose(`Muxing ${sources.length} streams to '${target}'`); + }) + .on('end', (stdout) => { + logger.verbose(`Muxed and saved '${target}'`); - resolve(stdout); - }) - .on('error', () => reject) - .save(target)) - .then(() => Promise.all(sources.map(source => fs.remove(source))).then(() => { - logger.verbose(`Cleaned up temporary files for '${target}'`); - })); + resolve(stdout); + }) + .on('error', () => reject) + .save(target); + }).then(() => Promise.all(sources.map((source) => fs.remove(source))).then(() => { + logger.verbose(`Cleaned up temporary files for '${target}'`); + })); } module.exports = mux; diff --git a/src/save/profileDetails.js b/src/save/profileDetails.js index fb3ebe5..be8b28f 100644 --- a/src/save/profileDetails.js +++ b/src/save/profileDetails.js @@ -40,7 +40,7 @@ async function saveProfileImage(user, args) { try { const { protocol, hostname, pathname } = new URL(image); - const stream = await fetchItem(`${protocol}//${hostname}${pathname}`, 0, { permalink: `https://reddit.com/user/${user.name}` }); + const { buffer: stream } = await fetchItem(`${protocol}//${hostname}${pathname}`, 0, { permalink: `https://reddit.com/user/${user.name}` }); const targets = await save(filepath, stream); return targets[0]; diff --git a/src/save/writeToIndex.js b/src/save/writeToIndex.js index c0a7ca8..2814480 100644 --- a/src/save/writeToIndex.js +++ b/src/save/writeToIndex.js @@ -8,6 +8,18 @@ const interpolate = require('../interpolate'); const save = require('./save'); const logger = require('../logger')(__filename); +function curatePosts(newPosts, indexedPosts) { + // console.log(indexedPosts, 'new'); + + const newIds = new Set(newPosts.map((post) => post.id)); + const uniqueIndexedPosts = indexedPosts.filter((post) => !newIds.has(post.id)); + const combinedPosts = newPosts.concat(uniqueIndexedPosts); + + // console.log(combinedPosts); + + return combinedPosts; +} + async function writeToIndex(posts, profilePaths, user, args) { const filepath = interpolate(config.library.index.file, null, null, null, null, user, false); const now = new Date(); @@ -24,6 +36,8 @@ async function writeToIndex(posts, profilePaths, user, args) { score: post.score, title: post.title, hash: post.hash, + phash: post.phash, + content: post.content, }; if (post.previewFallback) { @@ -34,11 +48,12 @@ async function writeToIndex(posts, profilePaths, user, args) { }); const data = { + updated: new Date(), profile: { image: profilePaths.image, description: profilePaths.description, }, - posts: newAndUpdatedEntries.concat(user.indexed.original), + posts: curatePosts(newAndUpdatedEntries, user.indexed.original), }; if (!data.profile.image && !data.profile.description && !data.posts.length) { @@ -46,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) { } try { - const yamlIndex = yaml.safeDump(data); + const yamlIndex = yaml.safeDump(data, { skipInvalid: true }); const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8')); logger.info(`Saved index with ${posts.length} new posts for ${user.name}`); diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index 847ee40..e5dd7f4 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -2,8 +2,8 @@ const Promise = require('bluebird'); -const getIndex = require('./getIndex.js'); -const curateUser = require('../curate/user.js'); +const getIndex = require('./getIndex'); +const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); const limiter = require('../limiter').reddit; @@ -23,7 +23,7 @@ async function getUser(username, reddit) { } } -const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { +const getPostsWrap = (reddit) => function getPosts(postIds, userPosts = {}) { return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch()); diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index cbe84f3..0d27200 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -2,9 +2,9 @@ const Promise = require('bluebird'); -const getIndex = require('./getIndex.js'); -const getArchivePostIds = require('../archives/getArchivePostIds.js'); -const curateUser = require('../curate/user.js'); +const getIndex = require('./getIndex'); +const getArchivePostIds = require('../archives/getArchivePostIds'); +const curateUser = require('../curate/user'); const logger = require('../logger')(__filename); const limiter = require('../limiter').reddit; @@ -46,9 +46,9 @@ async function getPosts(username, reddit, args) { } async function getArchivedPosts(username, posts, reddit) { - const postIds = await getArchivePostIds(username, posts.map(post => post.id)); + const postIds = await getArchivePostIds(username, posts.map((post) => post.id)); - return Promise.all(postIds.map(postId => limiter.schedule(async () => reddit.getSubmission(postId).fetch()))); + return Promise.all(postIds.map((postId) => limiter.schedule(async () => reddit.getSubmission(postId).fetch()))); } function getUserPostsWrap(reddit, args) {