From 7de0c51c166ff425737dfbdff2b32a1205c0c97a Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 11 Sep 2024 05:16:58 +0200 Subject: [PATCH] Fetching and storing comments for self posts. --- package-lock.json | 255 +++++++++++++++++++----------------- package.json | 2 +- src/app.js | 2 +- src/curate/hashPost.js | 7 +- src/curate/posts.js | 10 +- src/fetch/content.js | 40 +++++- src/fetch/info.js | 23 +++- src/methods/self.js | 40 +++++- src/save/textToStream.js | 4 +- src/sources/getPosts.js | 28 +++- src/sources/getUserPosts.js | 10 +- 11 files changed, 272 insertions(+), 149 deletions(-) diff --git a/package-lock.json b/package-lock.json index c3dfc60..a8f603e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "reddit-post-dump", + "name": "ripunzel", "version": "1.20.2", "lockfileVersion": 1, "requires": true, @@ -62,6 +62,7 @@ "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "dev": true, "requires": { "co": "^4.6.0", "fast-deep-equal": "^1.0.0", @@ -170,9 +171,9 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz", - "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "babel-code-frame": { "version": "6.26.0", @@ -240,14 +241,6 @@ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" }, - "boom": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", - "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", - "requires": { - "hoek": "4.x.x" - } - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -389,7 +382,8 @@ "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", + "dev": true }, "code-point-at": { "version": "1.1.0", @@ -467,24 +461,6 @@ "which": "^1.2.9" } }, - "cryptiles": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", - "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", - "requires": { - "boom": "5.x.x" - }, - "dependencies": { - "boom": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", - "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", - "requires": { - "hoek": "4.x.x" - } - } - } - }, "css-select": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", @@ -963,9 +939,9 @@ "optional": true }, "extend": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", - "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "external-editor": { "version": "2.2.0", @@ -986,7 +962,8 @@ "fast-deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", - "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=", + "dev": true }, "fast-json-stable-stringify": { "version": "2.0.0", @@ -1156,18 +1133,41 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", - "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "ajv": "^5.1.0", + "ajv": "^6.5.5", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", + "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + } } }, "harmony-reflect": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.0.tgz", - "integrity": "sha512-0kZ1XcoelFOLEjEtvWAZyq/1S55eDSieWEJwme311MNVNcRpvjlr2zA66kBV6WAB8C1XI1p1cXCnFPqd1BxlPg==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.1.tgz", + "integrity": "sha512-WJTeyp0JzGtHcuMsi7rw2VwtkvLa+JyfEKJCFyfcS0+CDkjQ5lHPu7zEhFZP+PDSRrEgXa5Ah0l1MbgbE41XjA==" }, "has": { "version": "1.0.1", @@ -1208,17 +1208,6 @@ "traverse": ">=0.2.4" } }, - "hawk": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", - "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", - "requires": { - "boom": "4.x.x", - "cryptiles": "3.x.x", - "hoek": "4.x.x", - "sntp": "2.x.x" - } - }, "hh-mm-ss": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz", @@ -1227,11 +1216,6 @@ "zero-fill": "^2.2.3" } }, - "hoek": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", - "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" - }, "hosted-git-info": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz", @@ -1677,7 +1661,8 @@ "json-schema-traverse": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=", + "dev": true }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -1906,9 +1891,9 @@ "integrity": "sha512-iGfd9Y6SFdTNldEy2L0GUhcarIutFmk+MPWIn9dmj8NMIup03G08uUF2KGbbmv/Ux4RT0VZJoP/sVbWA6d/VIw==" }, "oauth-sign": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", - "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "object-assign": { "version": "4.1.1", @@ -1967,11 +1952,6 @@ "wordwrap": "~1.0.0" } }, - "options": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", - "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" - }, "os-homedir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", @@ -2194,9 +2174,9 @@ "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "read-pkg": { "version": "2.0.0", @@ -2240,42 +2220,70 @@ "dev": true }, "request": { - "version": "2.85.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", - "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { "aws-sign2": "~0.7.0", - "aws4": "^1.6.0", + "aws4": "^1.8.0", "caseless": "~0.12.0", - "combined-stream": "~1.0.5", - "extend": "~3.0.1", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", "forever-agent": "~0.6.1", - "form-data": "~2.3.1", - "har-validator": "~5.0.3", - "hawk": "~6.0.2", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.17", - "oauth-sign": "~0.8.2", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", "performance-now": "^2.1.0", - "qs": "~6.5.1", - "safe-buffer": "^5.1.1", - "stringstream": "~0.0.5", - "tough-cookie": "~2.3.3", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", "tunnel-agent": "^0.6.0", - "uuid": "^3.1.0" + "uuid": "^3.3.2" + }, + "dependencies": { + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } + } } }, "request-promise": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-3.0.0.tgz", - "integrity": "sha1-vh7bJvQcSc0dVlbGdT1oQqEkn0Y=", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.4.tgz", + "integrity": "sha512-8wgMrvE546PzbR5WbYxUQogUnUDfM0S7QIFZMID+J73vdFARkFy+HElj4T+MWYhpXwlLp0EQ8Zoj8xUA0he4Vg==", "requires": { - "bluebird": "^3.3", - "lodash": "^4.6.1", - "request": "^2.34" + "bluebird": "^3.5.0", + "request-promise-core": "1.1.2", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" } }, "request-promise-core": { @@ -2443,23 +2451,28 @@ } }, "snoowrap": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/snoowrap/-/snoowrap-1.15.2.tgz", - "integrity": "sha512-h+JZA9edD4+cm3iuehucc/7VDhKuzY7EGqkHpPy+67OTpobswm6zVOHw5XrIzp9zsazvVu1Pg29NC/AhQ636AA==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/snoowrap/-/snoowrap-1.20.0.tgz", + "integrity": "sha512-TZ+hzHF+1aCCSchHabxGVEpP4qN73fqZS9kpBpsiiGAtogjPVlkIIJviGbxtyZ6/noXWlADqs/53eRcj0+Cl8g==", "requires": { - "bluebird": "^3.4.1", - "lodash": "^4.5.1", + "bluebird": "^3.5.5", + "lodash": "^4.17.15", "promise-chains": "^0.3.11", - "request-promise": "^3.0.0", - "ws": "^1.1.1" - } - }, - "sntp": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", - "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", - "requires": { - "hoek": "4.x.x" + "request": "^2.87.0", + "request-promise": "^4.2.2", + "ws": "^3.3.1" + }, + "dependencies": { + "bluebird": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.1.tgz", + "integrity": "sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg==" + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + } } }, "source-map": { @@ -2550,11 +2563,6 @@ "safe-buffer": "~5.1.0" } }, - "stringstream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", - "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" - }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -2693,9 +2701,9 @@ "dev": true }, "ultron": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.0.2.tgz", - "integrity": "sha1-rOEWq1V80Zc4ak6I9GhTeMiy5Po=" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", + "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" }, "universalify": { "version": "0.1.1", @@ -2728,9 +2736,9 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "uuid": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", - "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" }, "validate-npm-package-license": { "version": "3.0.3", @@ -2892,12 +2900,13 @@ } }, "ws": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz", - "integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", + "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", "requires": { - "options": ">=0.0.5", - "ultron": "1.0.x" + "async-limiter": "~1.0.0", + "safe-buffer": "~5.1.0", + "ultron": "~1.1.0" } }, "xml-name-validator": { diff --git a/package.json b/package.json index 54b6327..195e906 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "node-fetch": "^2.1.2", "object.omit": "^3.0.0", "object.pick": "^1.3.0", - "snoowrap": "^1.15.2", + "snoowrap": "^1.20.0", "template-format": "^1.2.4", "url-pattern": "^1.0.3", "yargs": "^11.0.0", diff --git a/src/app.js b/src/app.js index 15cd836..e1c2502 100644 --- a/src/app.js +++ b/src/app.js @@ -67,7 +67,7 @@ async function getCompletePosts() { const curatedUserPosts = curatePosts(userPosts, ignoreIds, args); - return attachContentInfo(curatedUserPosts); + return attachContentInfo(curatedUserPosts, reddit); } async function getDirectContent(links, ep) { diff --git a/src/curate/hashPost.js b/src/curate/hashPost.js index 428962a..88d1676 100644 --- a/src/curate/hashPost.js +++ b/src/curate/hashPost.js @@ -2,8 +2,9 @@ const crypto = require('crypto'); -const hashPost = post => { - return crypto.createHash('md5').update(post.id + post.subreddit_id + post.created_utc + post.title).digest('hex'); -}; +const hashPost = post => crypto + .createHash('md5') + .update(post.id + post.subreddit_id + post.created_utc + post.title) + .digest('hex'); module.exports = hashPost; diff --git a/src/curate/posts.js b/src/curate/posts.js index 5b06c92..8f9c83f 100644 --- a/src/curate/posts.js +++ b/src/curate/posts.js @@ -64,6 +64,8 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) score: post.score, preview: post.preview ? post.preview.images.map(image => image.source) : null, host, + direct: post.direct, + comments: post.comments, hash: hashPost(post), }; @@ -135,7 +137,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args) processed.add(host.id); } - return { ...acc, posts: [...acc.posts, curatedPost] }; + return { + ...acc, + posts: [ + ...acc.posts, + curatedPost, + ], + }; } const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts).reduce((accPosts, user) => { diff --git a/src/fetch/content.js b/src/fetch/content.js index 3aeebd4..c653427 100644 --- a/src/fetch/content.js +++ b/src/fetch/content.js @@ -11,10 +11,46 @@ const textToStream = require('../save/textToStream.js'); const saveMeta = require('../save/meta.js'); const mux = require('../save/mux.js'); const writeToIndex = require('../save/writeToIndex.js'); +const yaml = require('js-yaml'); + +function curateComments(comments) { + return comments.map((comment) => { + const curatedComment = { + text: comment.body, + author: comment.author, + score: comment.score, + date: comment.datetime, + }; + + if (comment.replies.length) { + return { + ...curatedComment, + replies: curateComments(comment.replies), + }; + } + + return curatedComment; + }); +} + +function selfPostToText(item, post) { + const curatedPost = { + title: post.title, + text: post.text, + author: post.user.name, + subreddit: post.subreddit, + score: post.score, + datetime: post.datetime, + url: post.url, + comments: curateComments(item.comments), + }; + + return yaml.safeDump(curatedPost); +} async function getStreams(item, post) { if (item.self) { - return [textToStream(post.text)]; + return [textToStream(selfPostToText(item, post))]; } const sources = item.mux ? [item.url].concat(item.mux) : [item.url]; @@ -97,7 +133,7 @@ async function fetchSaveUserContent(user, ep, args) { } async function fetchSaveDirectContent(content, host, ep) { - const data = await Promise.reduce(content.items, async (accItems, originalItem, index) => { + return Promise.reduce(content.items, async (accItems, originalItem, index) => { const item = { ...originalItem, index }; const streams = await getStreams(item, null); diff --git a/src/fetch/info.js b/src/fetch/info.js index 0cd88a8..ad05a1e 100644 --- a/src/fetch/info.js +++ b/src/fetch/info.js @@ -5,7 +5,7 @@ const Promise = require('bluebird'); const methods = require('../methods/methods.js'); -const attachContentInfo = users => Promise.reduce(Object.values(users), async (accUsers, user) => ({ +const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users), async (accUsers, user) => ({ ...accUsers, [user.name]: { ...user, @@ -17,14 +17,27 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a } try { - return [...accPosts, { ...post, content: await methods[post.host.method](post.host, post) }]; + return [ + ...accPosts, + { + ...post, + content: await methods[post.host.method](post.host, post, reddit), + }, + ]; } catch (error) { console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`); if (config.fetch.archives.preview && post.preview) { console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`); - return [...accPosts, { ...post, previewFallback: true, content: await methods.redditPreview(post.host, post) }]; + return [ + ...accPosts, + { + ...post, + previewFallback: true, + content: await methods.redditPreview(post.host, post), + }, + ]; } return accPosts; @@ -33,8 +46,8 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a }, }), {}); -async function getInfo(host) { - return methods[host.method](host); +async function getInfo(host, reddit) { + return methods[host.method](host, null, reddit); } module.exports = { diff --git a/src/methods/self.js b/src/methods/self.js index 3024fb8..3c0afe0 100644 --- a/src/methods/self.js +++ b/src/methods/self.js @@ -1,7 +1,39 @@ 'use strict'; -function self(host, post) { - console.log(post); +function curateComments(comments) { + return comments.map(comment => ({ + id: comment.id, + url: `https://reddit.com${comment.permalink}`, + author: comment.author.name, + body: comment.body, + html: comment.body_html, + score: comment.score, + datetime: new Date(comment.created * 1000), + edited: comment.edited, + controversiality: comment.controversiality, + gilded: comment.gilded, + stickied: comment.stickied, + distinguished: comment.distinguished, + locked: comment.locked, + archived: comment.archived, + parent: comment.parent_id, + replies: comment.replies ? curateComments(comment.replies) : [], + })); +} + +async function getFullPost(postId, reddit) { + return reddit + .getSubmission(postId) + .expandReplies({ + limit: Infinity, + depth: Infinity, + }) + .fetch(); +} + +async function self(host, originalPost, reddit) { + const post = await getFullPost(originalPost.id, reddit) || originalPost; + const curatedComments = curateComments(post.comments); return { album: null, @@ -10,11 +42,13 @@ function self(host, post) { url: post.url, title: post.title, text: post.text, + author: post.author.name, datetime: post.datetime, + comments: curatedComments, type: 'text/plain', self: true, original: post, - }] + }], }; } diff --git a/src/save/textToStream.js b/src/save/textToStream.js index 8de9bf9..f052662 100644 --- a/src/save/textToStream.js +++ b/src/save/textToStream.js @@ -1,6 +1,6 @@ 'use strict'; -const Readable = require('stream').Readable; +const { Readable } = require('stream'); function textToStream(text) { const stream = new Readable(); @@ -9,6 +9,6 @@ function textToStream(text) { stream.push(null); return stream; -}; +} module.exports = textToStream; diff --git a/src/sources/getPosts.js b/src/sources/getPosts.js index 50565e6..10f47ca 100644 --- a/src/sources/getPosts.js +++ b/src/sources/getPosts.js @@ -5,7 +5,7 @@ const Promise = require('bluebird'); const getIndex = require('./getIndex.js'); const curateUser = require('../curate/user.js'); -const getUser = async (username, reddit) => { +async function getUser(username, reddit) { try { const user = await reddit.getUser(username).fetch(); @@ -22,7 +22,9 @@ const getUser = async (username, reddit) => { const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => { - const post = await reddit.getSubmission(postId).fetch(); + const post = await reddit + .getSubmission(postId) + .fetch(); post.direct = true; @@ -34,13 +36,31 @@ const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) { // don't attempt to fetch deleted user if (post.author.name === '[deleted]') { - return { ...accUserPosts, '[deleted]': { name: '[deleted]', deleted: true, posts: [post] } }; + return { + ...accUserPosts, + '[deleted]': { + name: '[deleted]', + deleted: true, + posts: [post], + }, + }; } const user = await getUser(post.author.name, reddit); const { profile, posts: indexed } = await getIndex(user); - return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { profile, original: indexed, updated: [] } } }; + return { + ...accUserPosts, + [post.author.name]: { + ...user, + posts: [post], + indexed: { + profile, + original: indexed, + updated: [], + }, + }, + }; }), userPosts); }; diff --git a/src/sources/getUserPosts.js b/src/sources/getUserPosts.js index deb6e8e..f9ddc83 100644 --- a/src/sources/getUserPosts.js +++ b/src/sources/getUserPosts.js @@ -25,10 +25,12 @@ async function getUser(username, reddit) { async function getPosts(username, reddit, args) { try { - const submissions = await reddit.getUser(username).getSubmissions({ - sort: args.sort, - limit: Infinity, - }); + const submissions = await reddit + .getUser(username) + .getSubmissions({ + sort: args.sort, + limit: Infinity, + }); console.log(`Fetched ${submissions.length} submissions for '${username}' (https://reddit.com/user/${username})`);