DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for RedGifs and Reddit albums. Improved command line logger. Added rate limiters for reddit and host URLs.

This commit is contained in:
ThePendulum 2021-12-17 02:04:25 +01:00
parent 196f318316
commit 66e5bfa50a
33 changed files with 586 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintrc
View File

@ -6,6 +6,6 @@
"rules": { "rules": {
"no-console": 0, "no-console": 0,
"indent": ["error", 4], "indent": ["error", 4],
"max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}] "max-len": 0
} }
} }

3
README.md
View File

@ -12,9 +12,10 @@ Most features are optional and can easily be disabled!
* Extract single images from albums * Extract single images from albums
### Supported hosts ### Supported hosts
* Reddit text/self, images and videos[\*](#reddit-videos) * Reddit text/self, images, albums and videos[\*](#reddit-videos)
* Imgur (requires API key as of late 2019) * Imgur (requires API key as of late 2019)
* Gfycat * Gfycat
* RedGifs
* YouTube * YouTube
* PornHub, YouPorn, xHamster, RedTube, xnxx, YouJizz * PornHub, YouPorn, xHamster, RedTube, xnxx, YouJizz
* Twitter, Facebook, Instagram * Twitter, Facebook, Instagram

7
config/default.js
View File

@ -69,6 +69,13 @@ module.exports = {
reupload: [], reupload: [],
}, },
}, },
logger: {
level: 'info',
},
limiter: {
concurrency: 100,
interval: 100,
},
reddit: { reddit: {
api: { api: {
userAgent: 'ripunzel', userAgent: 'ripunzel',

213
package-lock.json generated
View File

@ -4,6 +4,16 @@
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
"@dabh/diagnostics": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz",
"integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==",
"requires": {
"colorspace": "1.1.x",
"enabled": "2.0.x",
"kuler": "^2.0.0"
}
},
"@types/node": { "@types/node": {
"version": "9.6.5", "version": "9.6.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.5.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.5.tgz",
@ -295,6 +305,11 @@
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
}, },
"bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
"brace-expansion": { "brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -443,6 +458,25 @@
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
}, },
"color": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
"integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==",
"requires": {
"color-convert": "^1.9.3",
"color-string": "^1.6.0"
},
"dependencies": {
"color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"requires": {
"color-name": "1.1.3"
}
}
}
},
"color-convert": { "color-convert": {
"version": "1.9.1", "version": "1.9.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
@ -455,8 +489,30 @@
"color-name": { "color-name": {
"version": "1.1.3", "version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
"dev": true },
"color-string": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.0.tgz",
"integrity": "sha512-9Mrz2AQLefkH1UvASKj6v6hj/7eWgjnT/cVsR8CumieLoT+g900exWeNogqtweI8dxloXN9BDQTYro1oWu/5CQ==",
"requires": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
}
},
"colors": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
},
"colorspace": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz",
"integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==",
"requires": {
"color": "^3.1.3",
"text-hex": "1.0.x"
}
}, },
"combined-stream": { "combined-stream": {
"version": "1.0.6", "version": "1.0.6",
@ -724,6 +780,11 @@
"jsbn": "~0.1.0" "jsbn": "~0.1.0"
} }
}, },
"enabled": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz",
"integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ=="
},
"end-of-stream": { "end-of-stream": {
"version": "1.4.4", "version": "1.4.4",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@ -1062,6 +1123,11 @@
"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc="
}, },
"fecha": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz",
"integrity": "sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q=="
},
"figures": { "figures": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
@ -1110,6 +1176,11 @@
"which": "^1.1.1" "which": "^1.1.1"
} }
}, },
"fn.name": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz",
"integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="
},
"foreach": { "foreach": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
@ -1161,6 +1232,11 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz",
"integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4="
}, },
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
},
"uuid": { "uuid": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
@ -1829,6 +1905,11 @@
"verror": "1.10.0" "verror": "1.10.0"
} }
}, },
"kuler": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz",
"integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="
},
"lcid": { "lcid": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
@ -1877,6 +1958,25 @@
"resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
"integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg="
}, },
"logform": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/logform/-/logform-2.3.0.tgz",
"integrity": "sha512-graeoWUH2knKbGthMtuG1EfaSPMZFZBIrhuJHhkS5ZseFBrc7DupCzihOQAzsK/qIKPQaPJ/lFQFctILUY5ARQ==",
"requires": {
"colors": "^1.2.1",
"fecha": "^4.2.0",
"ms": "^2.1.1",
"safe-stable-stringify": "^1.1.0",
"triple-beam": "^1.3.0"
},
"dependencies": {
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"lru-cache": { "lru-cache": {
"version": "4.1.2", "version": "4.1.2",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz",
@ -1905,9 +2005,9 @@
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
}, },
"mime": { "mime": {
"version": "1.6.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="
}, },
"mime-db": { "mime-db": {
"version": "1.33.0", "version": "1.33.0",
@ -2069,6 +2169,14 @@
"wrappy": "1" "wrappy": "1"
} }
}, },
"one-time": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz",
"integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==",
"requires": {
"fn.name": "1.x.x"
}
},
"onetime": { "onetime": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
@ -2538,6 +2646,11 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
}, },
"safe-stable-stringify": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-1.1.1.tgz",
"integrity": "sha512-ERq4hUjKDbJfE4+XtZLFPCDi8Vb1JqaxAPTxWFLBx8XcAlf9Bda/ZJdVezs/NAfsMQScyIlUMx+Yeu7P7rx5jw=="
},
"safer-buffer": { "safer-buffer": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@ -2580,6 +2693,21 @@
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
}, },
"simple-swizzle": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
"integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=",
"requires": {
"is-arrayish": "^0.3.1"
},
"dependencies": {
"is-arrayish": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
"integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
}
}
},
"slice-ansi": { "slice-ansi": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz",
@ -2672,6 +2800,11 @@
"tweetnacl": "~0.14.0" "tweetnacl": "~0.14.0"
} }
}, },
"stack-trace": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
"integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
},
"stealthy-require": { "stealthy-require": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
@ -2782,6 +2915,11 @@
"resolved": "https://registry.npmjs.org/template-format/-/template-format-1.2.4.tgz", "resolved": "https://registry.npmjs.org/template-format/-/template-format-1.2.4.tgz",
"integrity": "sha512-+8ItNMtMTBbsEHyPR1l7Ke1WZfl91PAcoTvwAcx5U28CRLd7ylzDLazv0kuDTzNmdq/RAOnsxFVWzr4QwVIFVg==" "integrity": "sha512-+8ItNMtMTBbsEHyPR1l7Ke1WZfl91PAcoTvwAcx5U28CRLd7ylzDLazv0kuDTzNmdq/RAOnsxFVWzr4QwVIFVg=="
}, },
"text-hex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
"integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
},
"text-table": { "text-table": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@ -2881,6 +3019,11 @@
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz",
"integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc="
}, },
"triple-beam": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
"integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw=="
},
"tunnel-agent": { "tunnel-agent": {
"version": "0.6.0", "version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
@ -3037,6 +3180,66 @@
"resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
"integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
}, },
"winston": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz",
"integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==",
"requires": {
"@dabh/diagnostics": "^2.0.2",
"async": "^3.1.0",
"is-stream": "^2.0.0",
"logform": "^2.2.0",
"one-time": "^1.0.0",
"readable-stream": "^3.4.0",
"stack-trace": "0.0.x",
"triple-beam": "^1.3.0",
"winston-transport": "^4.4.0"
},
"dependencies": {
"async": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/async/-/async-3.2.2.tgz",
"integrity": "sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g=="
},
"is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"winston-transport": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.1.tgz",
"integrity": "sha512-ciZRlU4CSjHqHe8RQG1iPxKMRVwv6ZJ0RC7DxStKWd0KjpAhPDy5gVYSCpIUq+5CUsP+IyNOTZy1X0tO2QZqjg==",
"requires": {
"logform": "^2.2.0",
"readable-stream": "^3.4.0",
"triple-beam": "^1.2.0"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"wordwrap": { "wordwrap": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",

3
package.json
View File

@ -32,6 +32,7 @@
"bhttp": "^1.2.4", "bhttp": "^1.2.4",
"blake2": "^4.0.0", "blake2": "^4.0.0",
"bluebird": "^3.5.1", "bluebird": "^3.5.1",
"bottleneck": "^2.19.5",
"cheerio": "^1.0.0-rc.2", "cheerio": "^1.0.0-rc.2",
"config": "^1.30.0", "config": "^1.30.0",
"date-fns": "^1.29.0", "date-fns": "^1.29.0",
@ -40,6 +41,7 @@
"fs-extra": "^5.0.0", "fs-extra": "^5.0.0",
"js-yaml": "^3.12.0", "js-yaml": "^3.12.0",
"jsdom": "^15.2.0", "jsdom": "^15.2.0",
"mime": "^3.0.0",
"mime-types": "^2.1.18", "mime-types": "^2.1.18",
"node-cron": "^1.2.1", "node-cron": "^1.2.1",
"node-exiftool": "^2.3.0", "node-exiftool": "^2.3.0",
@ -49,6 +51,7 @@
"snoowrap": "^1.20.0", "snoowrap": "^1.20.0",
"template-format": "^1.2.4", "template-format": "^1.2.4",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"winston": "^3.3.3",
"yargs": "^11.0.0", "yargs": "^11.0.0",
"youtube-dl": "^2.1.0" "youtube-dl": "^2.1.0"
}, },

24
src/app.js
View File

@ -7,21 +7,21 @@ const Promise = require('bluebird');
const exiftool = require('node-exiftool'); const exiftool = require('node-exiftool');
const exiftoolBin = require('dist-exiftool'); const exiftoolBin = require('dist-exiftool');
const cron = require('node-cron'); const cron = require('node-cron');
const { format } = require('date-fns');
require('array.prototype.flatten').shim(); require('array.prototype.flatten').shim();
const reddit = new Snoowrap(config.reddit.api); const reddit = new Snoowrap(config.reddit.api);
const args = require('./cli.js')(); const args = require('./cli')();
const logger = require('./logger')(__filename);
const dissectLink = require('./dissectLink.js'); const dissectLink = require('./dissectLink');
const curatePosts = require('./curate/posts.js'); const curatePosts = require('./curate/posts');
const { attachContentInfo, getInfo } = require('./fetch/info.js'); const { attachContentInfo, getInfo } = require('./fetch/info');
const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content.js'); const { fetchSaveUserContent, fetchSaveDirectContent } = require('./fetch/content');
const getPosts = require('./sources/getPosts.js')(reddit, args); const getPosts = require('./sources/getPosts')(reddit, args);
const getUserPosts = require('./sources/getUserPosts.js')(reddit, args); const getUserPosts = require('./sources/getUserPosts')(reddit, args);
async function getFileContents(location, label) { async function getFileContents(location, label) {
try { try {
@ -29,7 +29,7 @@ async function getFileContents(location, label) {
return fileContents.split('\n').filter(entry => entry && entry.slice(0, 1) !== '#'); return fileContents.split('\n').filter(entry => entry && entry.slice(0, 1) !== '#');
} catch (error) { } catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Could not read ${label} file '${location}': ${error}.`); logger.error(`Could not read ${label} file '${location}': ${error}.`);
return []; return [];
} }
@ -120,13 +120,13 @@ async function initApp() {
await ep.close(); await ep.close();
if (args.watch) { if (args.watch) {
console.log(`[${format(new Date(), 'YYYY-MM-DD HH:mm:ss')}] Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`); logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
} }
} catch (error) { } catch (error) {
if (args.debug) { if (args.debug) {
console.log('\x1b[31m%s\x1b[0m', error.stack); logger.error(error.stack);
} else { } else {
console.log('\x1b[31m%s\x1b[0m', error.message); logger.error(error.message);
} }
} }
} }

20
src/archives/getArchivePostIds.js
View File

@ -2,18 +2,20 @@
const config = require('config'); const config = require('config');
const archives = require('./archives.js'); const logger = require('../logger')(__filename);
const archives = require('./archives');
function getArchivePostIds(username, exclude) { function getArchivePostIds(username, exclude) {
console.log(`Finding archived posts for '${username}'...`); logger.info(`Finding archived posts for '${username}'...`);
return Promise.all(config.fetch.archives.reddit.map(source => archives[source](username))).then(postIds => postIds.flatten()).then(postIds => { return Promise.all(config.fetch.archives.reddit.map(source => archives[source](username)))
return exclude ? postIds.filter(postId => !exclude.includes(postId)) : postIds; .then(postIds => postIds.flatten())
}).then(postIds => { .then(postIds => (exclude ? postIds.filter(postId => !exclude.includes(postId)) : postIds))
console.log(`Found ${postIds.length} unique archived posts for user '${username}'`); .then((postIds) => {
logger.info(`Found ${postIds.length} unique archived posts for user '${username}'`);
return postIds; return postIds;
}); });
}; }
module.exports = getArchivePostIds; module.exports = getArchivePostIds;

16
src/cli.js
View File

@ -6,6 +6,22 @@ const yargs = require('yargs');
function getArgs() { function getArgs() {
const args = yargs const args = yargs
.command('npm start -- --user <username>') .command('npm start -- --user <username>')
.option('log-level', {
alias: 'level',
describe: 'CLI log verbosity',
type: 'string',
default: config.logger.level,
})
.option('interval', {
describe: 'Minimum wait time between HTTP requests',
type: 'number',
default: config.limiter.interval,
})
.option('concurrency', {
describe: 'Maximum HTTP requests pending at the same time',
type: 'number',
default: config.limiter.concurrency,
})
.option('users', { .option('users', {
alias: 'user', alias: 'user',
describe: 'Reddit usernames to fetch posts from', describe: 'Reddit usernames to fetch posts from',

45
src/curate/posts.js
View File

@ -7,43 +7,39 @@ const dissectLink = require('../dissectLink.js');
const hashPost = require('./hashPost.js'); const hashPost = require('./hashPost.js');
const { isAfter, isBefore, isEqual } = require('date-fns'); const { isAfter, isBefore, isEqual } = require('date-fns');
const logger = require('../logger')(__filename);
function report(curatedPosts, indexed, user, args) { function report(curatedPosts, indexed, user, args) {
const { const {
indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored, indexedUpdated, tooOldCount, tooRecentCount, beforeIndexedCount, afterIndexedCount, requestedIgnored, duplicates,
} = curatedPosts; } = curatedPosts;
if (indexedUpdated.length > 0) { if (indexedUpdated.length > 0) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`); logger.info(`Ignoring ${indexedUpdated.length} indexed posts for '${user.name}'`);
} }
if (requestedIgnored.length > 0) { if (requestedIgnored.length > 0) {
console.log( logger.info(`Ignoring ${requestedIgnored.length} posts because their IDs are specified to be ignored for '${user.name}'`);
'\x1b[33m%s\x1b[0m',
`Ignoring ${requestedIgnored.length} posts because their IDs are specified to be ignored for '${user.name}'`,
);
} }
if (tooOldCount > 0) { if (tooOldCount > 0) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`); logger.info(`Ignoring ${tooOldCount} older posts for '${user.name}' for specified date limit '${args.after}'`);
} }
if (tooRecentCount > 0) { if (tooRecentCount > 0) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${tooRecentCount} newer posts for '${user.name}' for specified date limit '${args.before}'`); logger.info(`Ignoring ${tooRecentCount} newer posts for '${user.name}' for specified date limit '${args.before}'`);
} }
if (beforeIndexedCount > 0) { if (beforeIndexedCount > 0) {
console.log( logger.info(`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`);
'\x1b[33m%s\x1b[0m',
`Ignoring ${beforeIndexedCount} posts older than the ${args.afterIndexed} indexed post (${indexed[args.afterIndexed].id}, ${indexed[args.afterIndexed].date}) for '${user.name}'`,
);
} }
if (afterIndexedCount > 0) { if (afterIndexedCount > 0) {
console.log( logger.info(`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`);
'\x1b[33m%s\x1b[0m', }
`Ignoring ${afterIndexedCount} posts newer than the ${args.beforeIndexed} indexed post (${indexed[args.beforeIndexed].id}, ${indexed[args.beforeIndexed].date}) for '${user.name}'`,
); if (duplicates.length > 0) {
logger.info(`Ignoring ${duplicates.length} duplicate posts for '${user.name}'`);
} }
} }
@ -103,7 +99,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
const ignoring = args.ignore ? args.ignore.find(prop => post[prop]) : null; const ignoring = args.ignore ? args.ignore.find(prop => post[prop]) : null;
if (ignoring) { if (ignoring) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring ${ignoring} post '${post.title}' (${permalink})`); logger.verbose(`Ignoring ${ignoring} post '${post.title}' (${permalink})`);
return acc; return acc;
} }
@ -117,21 +113,15 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
} }
if (hostIncludes || hostExcluded) { if (hostIncludes || hostExcluded) {
console.log( logger.info(`Ignoring source '${host.label}' from post '${post.url}' (${permalink})`);
'\x1b[33m%s\x1b[0m',
`Ignoring source '${host.label}' from post '${post.url}' (${permalink})`,
);
return acc; return acc;
} }
if (config.fetch.avoidDuplicates && processed.has(host.id)) { if (config.fetch.avoidDuplicates && processed.has(host.id)) {
console.log( logger.verbose(`Ignoring duplicate content '${post.url}' (cross-post, repost or superfluous --post ID) (${permalink})`);
'\x1b[33m%s\x1b[0m',
`Ignoring duplicate content '${post.url}' (cross-post, repost or superfluous --post ID) (${permalink})`,
);
return acc; return { ...acc, duplicates: [...acc.duplicates, curatedPost] };
} }
processed.add(host.id); processed.add(host.id);
@ -139,6 +129,7 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
return { return {
...acc, ...acc,
processed,
posts: [ posts: [
...acc.posts, ...acc.posts,
curatedPost, curatedPost,
@ -161,6 +152,8 @@ const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts
posts: [], posts: [],
indexedUpdated: [], indexedUpdated: [],
requestedIgnored: [], requestedIgnored: [],
duplicates: [],
duplicateCount: 0,
tooOldCount: 0, tooOldCount: 0,
tooRecentCount: 0, tooRecentCount: 0,
beforeIndexedCount: 0, beforeIndexedCount: 0,

12
src/dissectLink.js
View File

@ -18,6 +18,11 @@ const hosts = [
label: 'reddit', label: 'reddit',
pattern: new UrlPattern('http(s)\\://i.reddituploads.com/:id(?*)'), pattern: new UrlPattern('http(s)\\://i.reddituploads.com/:id(?*)'),
}, },
{
method: 'redditAlbum',
label: 'reddit',
pattern: new UrlPattern('http(s)\\://(www.)reddit.com/gallery/:id'),
},
{ {
method: 'redditVideo', method: 'redditVideo',
label: 'reddit', label: 'reddit',
@ -53,6 +58,11 @@ const hosts = [
label: 'gfycat', label: 'gfycat',
pattern: new UrlPattern('http(s)\\://(:server.)gfycat.com/(gifs/detail/)(:id-mobile)(:id-size_restricted)(:id)(.:ext)(?*)'), pattern: new UrlPattern('http(s)\\://(:server.)gfycat.com/(gifs/detail/)(:id-mobile)(:id-size_restricted)(:id)(.:ext)(?*)'),
}, },
{
method: 'redgifs',
label: 'redgifs',
pattern: new UrlPattern('http(s)\\://(:subdomain.)redgifs.com(/watch)/(:id-mobile)(:id)(.:ext)(?*)'),
},
{ {
method: 'erome', method: 'erome',
label: 'erome', label: 'erome',
@ -139,7 +149,5 @@ module.exports = function dissectLink(url) {
} }
} }
console.log(url);
return null; return null;
}; };

30
src/fetch/content.js
View File

@ -4,14 +4,15 @@ const config = require('config');
const Promise = require('bluebird'); const Promise = require('bluebird');
const yaml = require('js-yaml'); const yaml = require('js-yaml');
const saveProfileDetails = require('../save/profileDetails.js'); const logger = require('../logger')(__filename);
const fetchItem = require('./item.js'); const saveProfileDetails = require('../save/profileDetails');
const interpolate = require('../interpolate.js'); const fetchItem = require('./item');
const save = require('../save/save.js'); const interpolate = require('../interpolate');
// const textToStream = require('../save/textToStream.js'); const save = require('../save/save');
const saveMeta = require('../save/meta.js'); // const textToStream = require('../save/textToStream');
const mux = require('../save/mux.js'); const saveMeta = require('../save/meta');
const writeToIndex = require('../save/writeToIndex.js'); const mux = require('../save/mux');
const writeToIndex = require('../save/writeToIndex');
function curateComments(comments) { function curateComments(comments) {
return comments.map((comment) => { return comments.map((comment) => {
@ -107,6 +108,10 @@ async function fetchSaveUserContent(user, ep, args) {
const hashes = new Set(user.indexed.original.map(item => item.hash)); const hashes = new Set(user.indexed.original.map(item => item.hash));
const posts = await Promise.map(user.posts, async (post) => { const posts = await Promise.map(user.posts, async (post) => {
if (!post.content) {
return null;
}
const hash = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => { const hash = await Promise.reduce(post.content.items, async (accItems, originalItem, index) => {
const item = { ...originalItem, index }; const item = { ...originalItem, index };
const buffers = await getBuffers(item, post, post.host); const buffers = await getBuffers(item, post, post.host);
@ -118,10 +123,7 @@ async function fetchSaveUserContent(user, ep, args) {
// prevent duplicates // prevent duplicates
if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash)) { if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash)) {
console.log( logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`);
'\x1b[33m%s\x1b[0m',
`Ignoring duplicate file '${post.url}' (${post.permalink})`,
);
return buffers[0].hash; return buffers[0].hash;
} }
@ -148,12 +150,12 @@ async function fetchSaveUserContent(user, ep, args) {
concurrency: config.fetch.concurrency, concurrency: config.fetch.concurrency,
}); });
return writeToIndex(posts, profilePaths, user, args); return writeToIndex(posts.filter(Boolean), profilePaths, user, args);
} }
async function fetchSaveDirectContent(content, host, ep) { async function fetchSaveDirectContent(content, host, ep) {
return Promise.reduce(content.items, async (accItems, originalItem, index) => { return Promise.reduce(content.items, async (accItems, originalItem, index) => {
console.log(`Fetching and saving '${host.url}'`); logger.info(`Fetching and saving '${host.url}'`);
const item = { ...originalItem, index }; const item = { ...originalItem, index };
const buffers = await getBuffers(item, null, host); const buffers = await getBuffers(item, null, host);

11
src/fetch/info.js
View File

@ -3,7 +3,8 @@
const config = require('config'); const config = require('config');
const Promise = require('bluebird'); const Promise = require('bluebird');
const methods = require('../methods/methods.js'); const logger = require('../logger')(__filename);
const methods = require('../methods/methods');
const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users), async (accUsers, user) => ({ const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users), async (accUsers, user) => ({
...accUsers, ...accUsers,
@ -11,7 +12,7 @@ const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users)
...user, ...user,
posts: await Promise.reduce(user.posts, async (accPosts, post) => { posts: await Promise.reduce(user.posts, async (accPosts, post) => {
if (!post.host || !methods[post.host.method]) { if (!post.host || !methods[post.host.method]) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${post.url}' (${post.permalink})`); logger.warn(`Ignoring unsupported content '${post.url}' (${post.permalink})`);
return accPosts; return accPosts;
} }
@ -25,10 +26,10 @@ const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users)
}, },
]; ];
} catch (error) { } catch (error) {
console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`); logger.warn(`${error.message} (${post.permalink})`);
if (config.fetch.archives.preview && post.preview) { if (config.fetch.archives.preview && post.preview) {
console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`); logger.info(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
return [ return [
...accPosts, ...accPosts,
@ -53,7 +54,7 @@ async function getInfo(host, reddit, url) {
return info; return info;
} catch (error) { } catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring unsupported content '${url}'`); logger.verbose(`Ignoring unsupported content '${url}'`);
return null; return null;
} }

16
src/fetch/item.js
View File

@ -4,13 +4,14 @@ const config = require('config');
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const blake2 = require('blake2'); const blake2 = require('blake2');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').items;
async function fetchItem(url, attempt, post, host) { async function fetchItem(url, attempt, post, host) {
async function retry(error) { async function retry(error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch '${url}': ${error.message} (${post ? post.permalink : 'no post'})`); logger.warn(`Failed to fetch '${url}', ${attempt < config.fetch.retries ? 'retrying' : 'giving up'}: ${error.message} (${post ? post.permalink : 'no post'})`);
if (attempt < config.fetch.retries) { if (attempt < config.fetch.retries) {
console.log('Retrying...');
return fetchItem(url, attempt + 1, post); return fetchItem(url, attempt + 1, post);
} }
@ -18,13 +19,17 @@ async function fetchItem(url, attempt, post, host) {
} }
try { try {
const res = await bhttp.get(url); const res = await limiter.schedule(async () => bhttp.get(url));
if (!res.statusCode === 200) { if (!res.statusCode === 200) {
throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`); throw new Error(`Response not OK for '${url}', HTTP code '${res.status}'`);
} }
console.log(`Fetched '${host.url}' (${post ? post.permalink : 'no post'})`); if (!Buffer.isBuffer(res.body)) {
throw new Error(`Unexpected response for '${url}' (${res.status}): ${res.body}`);
}
logger.debug(`Fetched '${host ? host.url : url}' (${post ? post.permalink : 'no post'})`);
const hash = blake2.createHash('blake2b', { digestLength: 24 }); const hash = blake2.createHash('blake2b', { digestLength: 24 });
hash.update(res.body); hash.update(res.body);
@ -32,7 +37,6 @@ async function fetchItem(url, attempt, post, host) {
return Object.assign(res.body, { hash: contentHash }); return Object.assign(res.body, { hash: contentHash });
} catch (error) { } catch (error) {
console.log(error);
return retry(error); return retry(error);
} }
} }

19
src/limiter.js Normal file
View File

@ -0,0 +1,19 @@
'use strict';
const Bottleneck = require('bottleneck');
const args = require('./cli')();
module.exports = {
reddit: new Bottleneck({
reservoir: 30,
reservoirRefreshAmount: 30,
reservoirRefreshInterval: 60000,
maxConcurrent: 1,
minTime: 100,
}),
items: new Bottleneck({
maxConcurrent: args.concurrency,
minTime: args.interval,
}),
};

26
src/logger.js Normal file
View File

@ -0,0 +1,26 @@
'use strict';
const winston = require('winston');
const args = require('./cli.js')();
const logger = winston.createLogger({
level: args.logLevel,
transports: [
new winston.transports.Console({
level: args.logLevel,
format: winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format.colorize(),
winston.format.printf(info => `${info.timestamp} ${info.level}: ${info.message}`),
),
timestamp: true,
}),
],
});
function getLogger() {
return logger;
}
module.exports = getLogger;

4
src/methods/erome.js
View File

@ -4,6 +4,8 @@ const config = require('config');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const logger = require('../logger')(__filename);
const base = 'https://www.erome.com/'; const base = 'https://www.erome.com/';
async function erome(host, post) { async function erome(host, post) {
@ -50,7 +52,7 @@ async function erome(host, post) {
const extract = config.library.extractSingleAlbumItem && (items.length === 1); const extract = config.library.extractSingleAlbumItem && (items.length === 1);
if (extract) { if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${url}' (${post ? post.url : 'no post'})`); logger.verbose(`Extracting single item from album '${url}' (${post ? post.url : 'no post'})`);
} }
return { return {

25
src/methods/gfycat.js
View File

@ -1,13 +1,28 @@
'use strict'; 'use strict';
const fetch = require('node-fetch'); const bhttp = require('bhttp');
const redgifs = require('./redgifs');
async function gfycat(host) { async function gfycat(host) {
const res = await fetch(`https://api.gfycat.com/v1/gfycats/${host.id}`); const res = await bhttp.get(`https://api.gfycat.com/v1/gfycats/${host.id}`);
const data = await res.json(); const data = await res.body;
if (data.error) { if (data.errorMessage) {
throw new Error(data.error); const redirectRes = await bhttp.head(host.url, {
followRedirects: false,
});
if (redirectRes.statusCode === 301) {
// Gfycat redirects all NSFW gifs to RedGifs, likely the case
return redgifs({
...host,
url: `https://www.redgifs.com/watch/${host.id}`,
method: 'redgifs',
label: 'redgifs',
});
}
throw new Error(`Gfycat API returned error for source '${host.url}' (${res.status}): ${data.errorMessage}`);
} }
return { return {

47
src/methods/imgurAlbum.js
View File

@ -2,7 +2,8 @@
const config = require('config'); const config = require('config');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
// const mime = require('mime-types');
const logger = require('../logger')(__filename);
async function imgurAlbumApi(host, post) { async function imgurAlbumApi(host, post) {
const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, { const res = await fetch(`https://api.imgur.com/3/album/${host.id}`, {
@ -20,7 +21,7 @@ async function imgurAlbumApi(host, post) {
const extract = config.library.extractSingleAlbumItem && data.images.length === 1; const extract = config.library.extractSingleAlbumItem && data.images.length === 1;
if (extract) { if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${data.link}' (${post ? post.url : 'no post'})`); logger.verbose(`Extracting single item from album '${data.link}' (${post ? post.url : 'no post'})`);
} }
return { return {
@ -45,46 +46,4 @@ async function imgurAlbumApi(host, post) {
}; };
} }
/*
* as of late 2019, imgur requires log in to view albums and gallery images
async function imgurAlbum(host, post) {
const res = await fetch(`https://imgur.com/a/${post.host.id}`);
const html = await res.text();
if (res.status !== 200) {
if (config.methods.imgur.clientId) {
console.log('\x1b[31m%s\x1b[0m', `Could not fetch info for direct imgur album '${post.host.id}' (${res.statusText}), trying API fallback (${post.permalink})`);
return imgurAlbumApi(post);
}
throw new Error(`Could not fetch info for imgur album '${post.host.id}' (${res.statusText}) no API fallback configured`);
}
const dataString = html.replace(/\s+/g, ' ').match(/}}, item:(.*)}; var PREBID_TIMEOUT/)[1];
const data = JSON.parse(dataString);
const extract = config.library.album.extractSingleItem && data.album_images.images.length === 1;
return {
album: extract ? null : {
id: data.id,
url: `https://imgur.com/a/${post.host.id}`,
title: data.title,
description: data.description,
datetime: new Date(data.datetime),
},
items: data.album_images.images.map(item => ({
extracted: extract,
id: item.hash,
url: data.animated ? `https://i.imgur.com/${item.hash}.mp4` : `https://i.imgur.com/${item.hash}${item.ext}`,
title: item.title || (extract ? data.title : null),
description: item.description || (extract ? data.description : null),
type: item.animated ? 'video/mp4' : mime.lookup(item.ext.split('?')[0]),
datetime: new Date(item.datetime),
})),
};
}
*/
module.exports = imgurAlbumApi; module.exports = imgurAlbumApi;

35
src/methods/imgurImage.js
View File

@ -10,6 +10,10 @@ async function imgurImageApi(host) {
}, },
}); });
if (res.status !== 200) {
throw new Error(`Imgur API returned HTTP ${res.status} for source '${host.url}'`);
}
const { data } = await res.json(); const { data } = await res.json();
if (res.status !== 200) { if (res.status !== 200) {
@ -32,37 +36,6 @@ async function imgurImageApi(host) {
async function imgurImage(host, post) { async function imgurImage(host, post) {
return imgurImageApi(host, post); return imgurImageApi(host, post);
/*
* as of late 2019, imgur requires log in to view albums and gallery images
const res = await fetch(`https://imgur.com/${post.host.id}`);
const html = await res.text();
if (res.status !== 200) {
if (config.methods.imgur.clientId) {
console.log('\x1b[31m%s\x1b[0m', `Could not scrape info for imgur image '${post.host.id}' (${res.statusText}), trying API fallback (${post.permalink})`);
return imgurImageApi(post);
}
throw new Error(`Could not scrape info for imgur image '${post.host.id}' (${res.statusText}), no API fallback configured`);
}
const dataString = html.replace(/\s+/g, ' ').match(/}}, item:(.*)}; var PREBID_TIMEOUT/)[1];
const data = JSON.parse(dataString);
return {
album: null,
items: [{
id: data.hash,
url: data.animated ? `https://i.imgur.com/${post.host.id}.mp4` : `https://i.imgur.com/${post.host.id}${data.ext}`,
title: data.title,
description: data.description,
type: data.animated ? 'video/mp4' : data.mimetype,
datetime: new Date(data.timestamp || data.datetime),
}],
};
*/
} }
module.exports = imgurImage; module.exports = imgurImage;

4
src/methods/methods.js
View File

@ -7,8 +7,10 @@ const gfycat = require('./gfycat');
const imgurAlbum = require('./imgurAlbum'); const imgurAlbum = require('./imgurAlbum');
const imgurImage = require('./imgurImage'); const imgurImage = require('./imgurImage');
const redditImage = require('./redditImage'); const redditImage = require('./redditImage');
const redditAlbum = require('./redditAlbum');
const redditPreview = require('./redditPreview'); const redditPreview = require('./redditPreview');
const redditVideo = require('./redditVideo'); const redditVideo = require('./redditVideo');
const redgifs = require('./redgifs');
const self = require('./self'); const self = require('./self');
const vidbleAlbum = require('./vidbleAlbum'); const vidbleAlbum = require('./vidbleAlbum');
const vidbleImage = require('./vidbleImage'); const vidbleImage = require('./vidbleImage');
@ -23,8 +25,10 @@ module.exports = {
imgurAlbum, imgurAlbum,
imgurImage, imgurImage,
redditImage, redditImage,
redditAlbum,
redditPreview, redditPreview,
redditVideo, redditVideo,
redgifs,
self, self,
tube, tube,
vidbleAlbum, vidbleAlbum,

32
src/methods/redditAlbum.js Normal file
View File

@ -0,0 +1,32 @@
'use strict';
const mime = require('mime');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
async function redditAlbum(host, post) {
const res = await bhttp.get(host.url);
if (res.statusCode !== 200) {
throw new Error(res.body.toString());
}
const { document } = new JSDOM(res.body.toString(), { runScripts: 'dangerously' }).window;
const items = Array.from(document.querySelectorAll('li a'), el => el.href);
return {
album: {
id: host.id,
url: host.url,
title: post.title,
},
items: items.map(url => ({
id: new URL(url).pathname.match(/\/(.*).jpg/)[1],
url,
datetime: post.datetime,
type: mime.getType(url) || 'image/jpeg',
})),
};
}
module.exports = redditAlbum;

4
src/methods/redditImage.js
View File

@ -1,6 +1,6 @@
'use strict'; 'use strict';
const mime = require('mime-types'); const mime = require('mime');
async function redditImage(host, post) { async function redditImage(host, post) {
return { return {
@ -10,7 +10,7 @@ async function redditImage(host, post) {
url: post.url, url: post.url,
title: post.title, title: post.title,
datetime: post.datetime, datetime: post.datetime,
type: mime.lookup(post.url.split('/.')[0]) || 'image/jpeg', type: mime.getType(post.url) || 'image/jpeg',
original: post, original: post,
}], }],
}; };

69
src/methods/redgifs.js Normal file
View File

@ -0,0 +1,69 @@
'use strict';
const fetch = require('node-fetch');
const mime = require('mime');
function scrapeGallery(data) {
const oldestDate = Math.min(...data.gifs.map(gif => gif.createDate));
return {
album: {
id: data.id,
datetime: new Date(oldestDate * 1000),
},
items: data.gifs.map(gif => ({
id: gif.id,
url: gif.urls.hd,
description: gif.tags.join(', '),
type: mime.getType(gif.urls.hd),
datetime: new Date(gif.createDate * 1000),
original: gif,
})),
};
}
async function fetchGallery(galleryId) {
const res = await fetch(`https://api.redgifs.com/v2/gallery/${galleryId}`);
const data = await res.json();
if (!data.gifs) {
return null;
}
return scrapeGallery(data);
}
async function redgifs(host) {
const res = await fetch(`https://api.redgifs.com/v2/gifs/${host.id.toLowerCase()}`);
const data = await res.json();
if (data.errorMessage) {
throw new Error(`RedGifs API returned error for source '${host.url}' (${res.status}): ${data.errorMessage.description}`);
}
if (data.id && data.gifs) {
return scrapeGallery(data);
}
if (!data.gif) {
return null;
}
if (data.gif.gallery) {
return fetchGallery(data.gif.gallery);
}
return {
album: null,
items: [{
id: data.gif.id,
url: data.gif.urls.hd,
description: data.gif.tags.join(', '),
type: mime.getType(data.gif.urls.hd),
datetime: new Date(data.gif.createDate * 1000),
original: data.gif,
}],
};
}
module.exports = redgifs;

6
src/methods/tube.js
View File

@ -3,6 +3,8 @@
const youtubedl = require('youtube-dl'); const youtubedl = require('youtube-dl');
const dateFns = require('date-fns'); const dateFns = require('date-fns');
const logger = require('../logger')(__filename);
async function tube(host, post) { async function tube(host, post) {
try { try {
const data = await new Promise((resolve, reject) => { const data = await new Promise((resolve, reject) => {
@ -15,7 +17,7 @@ async function tube(host, post) {
}); });
}); });
host.id = data.display_id; host.id = data.display_id; // eslint-disable-line no-param-reassign
return { return {
album: null, album: null,
@ -32,7 +34,7 @@ async function tube(host, post) {
], ],
}; };
} catch (error) { } catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring possible profile page '${host.url}' (${post ? post.permalink : 'no post'})`); logger.warn(`Ignoring possible image or profile page '${host.url}' (${post ? post.permalink : 'no post'})`);
return null; return null;
} }

6
src/methods/vidbleAlbum.js
View File

@ -6,6 +6,8 @@ const UrlPattern = require('url-pattern');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const mime = require('mime-types'); const mime = require('mime-types');
const logger = require('../logger')(__filename);
const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)'); const pattern = new UrlPattern('https\\://(www.)vidble.com/:id(_med)(.:ext)');
async function vidbleAlbum(host, post) { async function vidbleAlbum(host, post) {
@ -23,7 +25,7 @@ async function vidbleAlbum(host, post) {
const extract = config.library.extractSingleAlbumItem && imgUrls.length === 1; const extract = config.library.extractSingleAlbumItem && imgUrls.length === 1;
if (extract) { if (extract) {
console.log('\x1b[36m%s\x1b[0m', `Extracting single item from album '${post.title}' - ${res.link}`); logger.verbose(`Extracting single item from album '${post.title}' - ${res.link}`);
} }
return { return {
@ -43,7 +45,7 @@ async function vidbleAlbum(host, post) {
id, id,
url: `https://vidble.com/${id}.${components.ext}`, url: `https://vidble.com/${id}.${components.ext}`,
type: mimetype, type: mimetype,
datetime: post.datetime, datetime: post ? post.datetime : null,
}; };
}), }),
}; };

4
src/save/meta.js
View File

@ -1,9 +1,11 @@
'use strict'; 'use strict';
const logger = require('../logger')(__filename);
async function saveMeta(filepath, meta, ep) { async function saveMeta(filepath, meta, ep) {
await ep.writeMetadata(filepath, meta, ['overwrite_original']); await ep.writeMetadata(filepath, meta, ['overwrite_original']);
console.log('\x1b[36m%s\x1b[0m', `Wrote metadata to '${filepath}'`); logger.debug(`Wrote metadata to '${filepath}'`);
} }
module.exports = saveMeta; module.exports = saveMeta;

35
src/save/mux.js
View File

@ -3,24 +3,25 @@
const ffmpeg = require('fluent-ffmpeg'); const ffmpeg = require('fluent-ffmpeg');
const fs = require('fs-extra'); const fs = require('fs-extra');
function mux(target, sources, item) { const logger = require('../logger')(__filename);
return new Promise((resolve, reject) => {
return sources.reduce((acc, source) => { function mux(target, sources) {
return acc.input(source); return new Promise((resolve, reject) => sources.reduce((acc, source) => acc.input(source), ffmpeg())
}, ffmpeg()).videoCodec('copy').audioCodec('copy').on('start', cmd => { .videoCodec('copy')
console.log('\x1b[36m%s\x1b[0m', `Muxing ${sources.length} streams to '${target}'`); .audioCodec('copy')
}).on('end', (stdout) => { .on('start', () => {
console.log('\x1b[32m%s\x1b[0m', `Muxed and saved '${target}'`); logger.verbose(`Muxing ${sources.length} streams to '${target}'`);
})
.on('end', (stdout) => {
logger.verbose(`Muxed and saved '${target}'`);
resolve(stdout); resolve(stdout);
}).on('error', error => reject).save(target); })
}).then(() => { .on('error', () => reject)
return Promise.all(sources.map(source => { .save(target))
return fs.remove(source); .then(() => Promise.all(sources.map(source => fs.remove(source))).then(() => {
})).then(() => { logger.verbose(`Cleaned up temporary files for '${target}'`);
console.log('\x1b[36m%s\x1b[0m', `Cleaned up temporary files for '${target}'`); }));
}); }
});
};
module.exports = mux; module.exports = mux;

21
src/save/profileDetails.js
View File

@ -4,14 +4,15 @@ const config = require('config');
const Promise = require('bluebird'); const Promise = require('bluebird');
const UrlPattern = require('url-pattern'); const UrlPattern = require('url-pattern');
const interpolate = require('../interpolate.js'); const interpolate = require('../interpolate');
const fetchItem = require('../fetch/item.js'); const fetchItem = require('../fetch/item');
// const textToStream = require('./textToStream.js'); // const textToStream = require('./textToStream');
const save = require('./save.js'); const save = require('./save');
const logger = require('../logger')(__filename);
async function saveProfileImage(user, args) { async function saveProfileImage(user, args) {
if (!args.redownloadProfile && user.indexed.profile.image) { if (!args.redownloadProfile && user.indexed.profile.image) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile image for '${user.name}' (https://reddit.com/user/${user.name})`); logger.verbose(`Ignoring already present profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
return user.indexed.profile.image; return user.indexed.profile.image;
} }
@ -20,7 +21,7 @@ async function saveProfileImage(user, args) {
const image = user.profile ? user.profile.image : user.image; const image = user.profile ? user.profile.image : user.image;
if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) { if (config.library.profile.avoidAvatar && new UrlPattern('http(s)\\://(www.)redditstatic.com/avatars/:id(.:ext)(?:query)').match(image)) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring standard avatar profile image for '${user.name}' (https://reddit.com/user/${user.name})`); logger.verbose(`Ignoring standard avatar profile image for '${user.name}' (https://reddit.com/user/${user.name})`);
return null; return null;
} }
@ -44,7 +45,7 @@ async function saveProfileImage(user, args) {
return targets[0]; return targets[0];
} catch (error) { } catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile image for '${user.name}': ${error} (https://reddit.com/user/${user.name})`); logger.warn(`Could not save profile image for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
return null; return null;
} }
@ -55,7 +56,7 @@ async function saveProfileImage(user, args) {
async function saveProfileDescription(user, args) { async function saveProfileDescription(user, args) {
if (!args.redownloadProfile && user.indexed.profile.description) { if (!args.redownloadProfile && user.indexed.profile.description) {
console.log('\x1b[33m%s\x1b[0m', `Ignoring already present profile description for '${user.name}' (https://reddit.com/user/${user.name})`); logger.verbose(`Ignoring already present profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
return user.indexed.profile.description; return user.indexed.profile.description;
} }
@ -70,13 +71,13 @@ async function saveProfileDescription(user, args) {
return targets[0]; return targets[0];
} catch (error) { } catch (error) {
console.log('\x1b[33m%s\x1b[0m', `Could not save profile description for '${user.name}': ${error} (https://reddit.com/user/${user.name})`); logger.error(`Could not save profile description for '${user.name}': ${error} (https://reddit.com/user/${user.name})`);
return null; return null;
} }
} }
console.log('\x1b[33m%s\x1b[0m', `No profile description for '${user.name}' (https://reddit.com/user/${user.name})`); logger.verbose(`No profile description for '${user.name}' (https://reddit.com/user/${user.name})`);
return null; return null;
} }

39
src/save/save.js
View File

@ -4,6 +4,7 @@ const config = require('config');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const truncate = require('../utils/truncate-bytes'); const truncate = require('../utils/truncate-bytes');
const logger = require('../logger')(__filename);
function limitPathElement(element, limit) { function limitPathElement(element, limit) {
return element.split('/').map((component) => { return element.split('/').map((component) => {
@ -30,48 +31,14 @@ async function writeBufferToFile(target, buffer, item) {
await fs.writeFile(target, buffer); await fs.writeFile(target, buffer);
if (item && item.mux) { if (item && item.mux) {
console.log(`Temporarily saved '${target}', queued for muxing`); logger.debug(`Temporarily saved '${target}', queued for muxing`);
} else { } else {
console.log('\x1b[32m%s\x1b[0m', `Saved '${target}'`); logger.verbose(`Saved '${target}'`);
} }
return target; return target;
} }
/*
async function pipeStreamToFile(target, stream, item) {
const file = fs.createWriteStream(target);
return new Promise((resolve, reject) => {
stream.pipe(file);
stream.on('error', reject);
stream.on('end', () => {
if (item && item.mux) {
console.log(`Temporarily saved '${target}', queued for muxing`);
} else {
console.log('\x1b[32m%s\x1b[0m', `Saved '${target}'`);
}
resolve(target);
});
});
}
async function save(requestedFilepath, streamOrStreams, item) {
const pathElements = getPathElements(requestedFilepath);
const streams = [].concat(streamOrStreams); // allow for single stream argument
await fs.ensureDir(pathElements.dir);
return Promise.all(streams.map((stream, index) => {
const target = path.join(pathElements.root, pathElements.dir, `${pathElements.name}${streams.length > 1 ? `-${index}` : ''}${pathElements.ext}`);
return pipeStreamToFile(target, stream, item);
}));
}
*/
async function save(requestedFilepath, bufferOrBuffers, item) { async function save(requestedFilepath, bufferOrBuffers, item) {
const pathElements = getPathElements(requestedFilepath); const pathElements = getPathElements(requestedFilepath);
const buffers = [].concat(bufferOrBuffers); // allow for single stream argument const buffers = [].concat(bufferOrBuffers); // allow for single stream argument

11
src/save/writeToIndex.js
View File

@ -6,6 +6,7 @@ const yaml = require('js-yaml');
const interpolate = require('../interpolate'); const interpolate = require('../interpolate');
// const textToStream = require('./textToStream'); // const textToStream = require('./textToStream');
const save = require('./save'); const save = require('./save');
const logger = require('../logger')(__filename);
async function writeToIndex(posts, profilePaths, user, args) { async function writeToIndex(posts, profilePaths, user, args) {
const filepath = interpolate(config.library.index.file, null, null, null, null, user, false); const filepath = interpolate(config.library.index.file, null, null, null, null, user, false);
@ -44,8 +45,14 @@ async function writeToIndex(posts, profilePaths, user, args) {
return false; return false;
} }
// return save(filepath, textToStream(yaml.safeDump(data))); try {
return save(filepath, Buffer.from(yaml.safeDump(data), 'utf8')); const yamlIndex = yaml.safeDump(data);
return save(filepath, Buffer.from(yamlIndex, 'utf8'));
} catch (error) {
logger.error(`Could not save index for ${user.username}: ${error.message}`);
return null;
}
} }
module.exports = writeToIndex; module.exports = writeToIndex;

5
src/sources/getIndex.js
View File

@ -4,7 +4,8 @@ const config = require('config');
const fs = require('fs-extra'); const fs = require('fs-extra');
const yaml = require('js-yaml'); const yaml = require('js-yaml');
const interpolate = require('../interpolate.js'); const logger = require('../logger')(__filename);
const interpolate = require('../interpolate');
async function getIndex(user) { async function getIndex(user) {
const indexFilePath = interpolate(config.library.index.file, null, null, null, null, user, false); const indexFilePath = interpolate(config.library.index.file, null, null, null, null, user, false);
@ -14,7 +15,7 @@ async function getIndex(user) {
return yaml.safeLoad(indexFile); return yaml.safeLoad(indexFile);
} catch (error) { } catch (error) {
console.log('\x1b[33m%s\x1b[0m', `No index file found for '${user.name}' at '${indexFilePath}'`); logger.info(`No index file found for '${user.name}' at '${indexFilePath}'`);
return { profile: { image: null, description: null }, posts: [] }; return { profile: { image: null, description: null }, posts: [] };
} }

23
src/sources/getPosts.js
View File

@ -5,33 +5,38 @@ const Promise = require('bluebird');
const getIndex = require('./getIndex.js'); const getIndex = require('./getIndex.js');
const curateUser = require('../curate/user.js'); const curateUser = require('../curate/user.js');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').reddit;
async function getUser(username, reddit) { async function getUser(username, reddit) {
try { try {
const user = await reddit.getUser(username).fetch(); const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
return curateUser(user); return curateUser(user);
} catch (error) { } catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return { return {
name: username, name: username,
fallback: true, fallback: true,
}; };
} }
}; }
const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
const post = await reddit const post = await limiter.schedule(async () => reddit.getSubmission(postId).fetch());
.getSubmission(postId)
.fetch();
post.direct = true; post.direct = true;
if (accUserPosts[post.author.name]) { if (accUserPosts[post.author.name]) {
accUserPosts[post.author.name].posts = accUserPosts[post.author.name].posts.concat(post); return {
...accUserPosts,
return accUserPosts; [post.author.name]: {
...accUserPosts[post.author.name],
posts: [...accUserPosts[post.author.name].posts, post],
},
};
} }
// don't attempt to fetch deleted user // don't attempt to fetch deleted user

25
src/sources/getUserPosts.js
View File

@ -6,15 +6,18 @@ const getIndex = require('./getIndex.js');
const getArchivePostIds = require('../archives/getArchivePostIds.js'); const getArchivePostIds = require('../archives/getArchivePostIds.js');
const curateUser = require('../curate/user.js'); const curateUser = require('../curate/user.js');
const logger = require('../logger')(__filename);
const limiter = require('../limiter').reddit;
async function getUser(username, reddit) { async function getUser(username, reddit) {
try { try {
const user = await reddit.getUser(username).fetch(); const user = await limiter.schedule(async () => reddit.getUser(username).fetch());
console.log(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`); logger.info(`Fetched user profile for '${username}' (https://reddit.com/user/${username})`);
return curateUser(user); return curateUser(user);
} catch (error) { } catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); logger.error(`Failed to fetch reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return { return {
name: username, name: username,
@ -25,18 +28,18 @@ async function getUser(username, reddit) {
async function getPosts(username, reddit, args) { async function getPosts(username, reddit, args) {
try { try {
const submissions = await reddit const submissions = await limiter.schedule(async () => reddit
.getUser(username) .getUser(username)
.getSubmissions({ .getSubmissions({
sort: args.sort, sort: args.sort,
limit: Infinity, limit: Infinity,
}); }));
console.log(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`); logger.info(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);
return submissions; return submissions;
} catch (error) { } catch (error) {
console.log('\x1b[31m%s\x1b[0m', `Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`); logger.warn(`Failed to fetch posts from reddit user '${username}': ${error.message} (https://reddit.com/user/${username})`);
return []; return [];
} }
@ -45,7 +48,7 @@ async function getPosts(username, reddit, args) {
async function getArchivedPosts(username, posts, reddit) { async function getArchivedPosts(username, posts, reddit) {
const postIds = await getArchivePostIds(username, posts.map(post => post.id)); const postIds = await getArchivePostIds(username, posts.map(post => post.id));
return Promise.all(postIds.map(postId => reddit.getSubmission(postId).fetch())); return Promise.all(postIds.map(postId => limiter.schedule(async () => reddit.getSubmission(postId).fetch())));
} }
function getUserPostsWrap(reddit, args) { function getUserPostsWrap(reddit, args) {
@ -78,12 +81,12 @@ function getUserPostsWrap(reddit, args) {
return null; return null;
} catch (error) { } catch (error) {
console.log(username, error); logger.error(`Failed to fetch posts from 'username': ${error.message}`);
return null; return null;
} }
}, { }, {
concurrency: 5, concurrency: 10,
}); });
return users.reduce( return users.reduce(
@ -97,7 +100,7 @@ function getUserPostsWrap(reddit, args) {
{}, {},
); );
} catch (error) { } catch (error) {
console.log(error); logger.error(error);
throw error; throw error;
} }