DebaucheryLibrarian
/
ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fetching and storing comments for self posts.

This commit is contained in:
ThePendulum 2019-11-05 01:58:00 +01:00
parent 804867626f
commit e175ce38c8
11 changed files with 272 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
package-lock.json generated
View File

@ -1,5 +1,5 @@
{
"name": "reddit-post-dump",
"name": "ripunzel",
"version": "1.20.2",
"lockfileVersion": 1,
"requires": true,
@ -62,6 +62,7 @@
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
"integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
"dev": true,
"requires": {
"co": "^4.6.0",
"fast-deep-equal": "^1.0.0",
@ -170,9 +171,9 @@
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
},
"aws4": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz",
"integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w=="
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
},
"babel-code-frame": {
"version": "6.26.0",
@ -240,14 +241,6 @@
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
},
"boom": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
"integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=",
"requires": {
"hoek": "4.x.x"
}
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -389,7 +382,8 @@
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
"dev": true
},
"code-point-at": {
"version": "1.1.0",
@ -467,24 +461,6 @@
"which": "^1.2.9"
}
},
"cryptiles": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz",
"integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=",
"requires": {
"boom": "5.x.x"
},
"dependencies": {
"boom": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz",
"integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==",
"requires": {
"hoek": "4.x.x"
}
}
}
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@ -963,9 +939,9 @@
"optional": true
},
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"external-editor": {
"version": "2.2.0",
@ -986,7 +962,8 @@
"fast-deep-equal": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
"integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
"integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=",
"dev": true
},
"fast-json-stable-stringify": {
"version": "2.0.0",
@ -1156,18 +1133,41 @@
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
},
"har-validator": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
"integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
"version": "5.1.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
"integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
"requires": {
"ajv": "^5.1.0",
"ajv": "^6.5.5",
"har-schema": "^2.0.0"
},
"dependencies": {
"ajv": {
"version": "6.10.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz",
"integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==",
"requires": {
"fast-deep-equal": "^2.0.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
}
}
},
"harmony-reflect": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.0.tgz",
"integrity": "sha512-0kZ1XcoelFOLEjEtvWAZyq/1S55eDSieWEJwme311MNVNcRpvjlr2zA66kBV6WAB8C1XI1p1cXCnFPqd1BxlPg=="
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.1.tgz",
"integrity": "sha512-WJTeyp0JzGtHcuMsi7rw2VwtkvLa+JyfEKJCFyfcS0+CDkjQ5lHPu7zEhFZP+PDSRrEgXa5Ah0l1MbgbE41XjA=="
},
"has": {
"version": "1.0.1",
@ -1208,17 +1208,6 @@
"traverse": ">=0.2.4"
}
},
"hawk": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
"integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==",
"requires": {
"boom": "4.x.x",
"cryptiles": "3.x.x",
"hoek": "4.x.x",
"sntp": "2.x.x"
}
},
"hh-mm-ss": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz",
@ -1227,11 +1216,6 @@
"zero-fill": "^2.2.3"
}
},
"hoek": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz",
"integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA=="
},
"hosted-git-info": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz",
@ -1677,7 +1661,8 @@
"json-schema-traverse": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
"integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
"integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=",
"dev": true
},
"json-stable-stringify-without-jsonify": {
"version": "1.0.1",
@ -1906,9 +1891,9 @@
"integrity": "sha512-iGfd9Y6SFdTNldEy2L0GUhcarIutFmk+MPWIn9dmj8NMIup03G08uUF2KGbbmv/Ux4RT0VZJoP/sVbWA6d/VIw=="
},
"oauth-sign": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
},
"object-assign": {
"version": "4.1.1",
@ -1967,11 +1952,6 @@
"wordwrap": "~1.0.0"
}
},
"options": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz",
"integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8="
},
"os-homedir": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
@ -2194,9 +2174,9 @@
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"qs": {
"version": "6.5.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
"integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
"version": "6.5.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
},
"read-pkg": {
"version": "2.0.0",
@ -2240,42 +2220,70 @@
"dev": true
},
"request": {
"version": "2.85.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz",
"integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==",
"version": "2.88.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
"requires": {
"aws-sign2": "~0.7.0",
"aws4": "^1.6.0",
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.5",
"extend": "~3.0.1",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.1",
"har-validator": "~5.0.3",
"hawk": "~6.0.2",
"form-data": "~2.3.2",
"har-validator": "~5.1.0",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.17",
"oauth-sign": "~0.8.2",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.1",
"safe-buffer": "^5.1.1",
"stringstream": "~0.0.5",
"tough-cookie": "~2.3.3",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.4.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.1.0"
"uuid": "^3.3.2"
},
"dependencies": {
"mime-db": {
"version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
"integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA=="
},
"mime-types": {
"version": "2.1.24",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
"integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
"requires": {
"mime-db": "1.40.0"
}
},
"safe-buffer": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
"integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg=="
},
"tough-cookie": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
"integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
"requires": {
"psl": "^1.1.24",
"punycode": "^1.4.1"
}
}
}
},
"request-promise": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-3.0.0.tgz",
"integrity": "sha1-vh7bJvQcSc0dVlbGdT1oQqEkn0Y=",
"version": "4.2.4",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.4.tgz",
"integrity": "sha512-8wgMrvE546PzbR5WbYxUQogUnUDfM0S7QIFZMID+J73vdFARkFy+HElj4T+MWYhpXwlLp0EQ8Zoj8xUA0he4Vg==",
"requires": {
"bluebird": "^3.3",
"lodash": "^4.6.1",
"request": "^2.34"
"bluebird": "^3.5.0",
"request-promise-core": "1.1.2",
"stealthy-require": "^1.1.1",
"tough-cookie": "^2.3.3"
}
},
"request-promise-core": {
@ -2443,23 +2451,28 @@
}
},
"snoowrap": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/snoowrap/-/snoowrap-1.15.2.tgz",
"integrity": "sha512-h+JZA9edD4+cm3iuehucc/7VDhKuzY7EGqkHpPy+67OTpobswm6zVOHw5XrIzp9zsazvVu1Pg29NC/AhQ636AA==",
"version": "1.20.0",
"resolved": "https://registry.npmjs.org/snoowrap/-/snoowrap-1.20.0.tgz",
"integrity": "sha512-TZ+hzHF+1aCCSchHabxGVEpP4qN73fqZS9kpBpsiiGAtogjPVlkIIJviGbxtyZ6/noXWlADqs/53eRcj0+Cl8g==",
"requires": {
"bluebird": "^3.4.1",
"lodash": "^4.5.1",
"bluebird": "^3.5.5",
"lodash": "^4.17.15",
"promise-chains": "^0.3.11",
"request-promise": "^3.0.0",
"ws": "^1.1.1"
}
"request": "^2.87.0",
"request-promise": "^4.2.2",
"ws": "^3.3.1"
},
"sntp": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz",
"integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==",
"requires": {
"hoek": "4.x.x"
"dependencies": {
"bluebird": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.1.tgz",
"integrity": "sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg=="
},
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
}
}
},
"source-map": {
@ -2550,11 +2563,6 @@
"safe-buffer": "~5.1.0"
}
},
"stringstream": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
"integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
},
"strip-ansi": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
@ -2693,9 +2701,9 @@
"dev": true
},
"ultron": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.0.2.tgz",
"integrity": "sha1-rOEWq1V80Zc4ak6I9GhTeMiy5Po="
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
},
"universalify": {
"version": "0.1.1",
@ -2728,9 +2736,9 @@
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"uuid": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
"integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz",
"integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ=="
},
"validate-npm-package-license": {
"version": "3.0.3",
@ -2892,12 +2900,13 @@
}
},
"ws": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz",
"integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==",
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"requires": {
"options": ">=0.0.5",
"ultron": "1.0.x"
"async-limiter": "~1.0.0",
"safe-buffer": "~5.1.0",
"ultron": "~1.1.0"
}
},
"xml-name-validator": {

2
package.json
View File

@ -44,7 +44,7 @@
"node-fetch": "^2.1.2",
"object.omit": "^3.0.0",
"object.pick": "^1.3.0",
"snoowrap": "^1.15.2",
"snoowrap": "^1.20.0",
"template-format": "^1.2.4",
"url-pattern": "^1.0.3",
"yargs": "^11.0.0",

2
src/app.js
View File

@ -67,7 +67,7 @@ async function getCompletePosts() {
const curatedUserPosts = curatePosts(userPosts, ignoreIds, args);
return attachContentInfo(curatedUserPosts);
return attachContentInfo(curatedUserPosts, reddit);
}
async function getDirectContent(links, ep) {

7
src/curate/hashPost.js
View File

@ -2,8 +2,9 @@
const crypto = require('crypto');
const hashPost = post => {
return crypto.createHash('md5').update(post.id + post.subreddit_id + post.created_utc + post.title).digest('hex');
};
const hashPost = post => crypto
.createHash('md5')
.update(post.id + post.subreddit_id + post.created_utc + post.title)
.digest('hex');
module.exports = hashPost;

10
src/curate/posts.js
View File

@ -64,6 +64,8 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
score: post.score,
preview: post.preview ? post.preview.images.map(image => image.source) : null,
host,
direct: post.direct,
comments: post.comments,
hash: hashPost(post),
};
@ -135,7 +137,13 @@ function curatePost(acc, post, user, index, indexed, ignoreIds, processed, args)
processed.add(host.id);
}
return { ...acc, posts: [...acc.posts, curatedPost] };
return {
...acc,
posts: [
...acc.posts,
curatedPost,
],
};
}
const curatePosts = (userPosts, ignoreIdsArray, args) => Object.values(userPosts).reduce((accPosts, user) => {

40
src/fetch/content.js
View File

@ -11,10 +11,46 @@ const textToStream = require('../save/textToStream.js');
const saveMeta = require('../save/meta.js');
const mux = require('../save/mux.js');
const writeToIndex = require('../save/writeToIndex.js');
const yaml = require('js-yaml');
function curateComments(comments) {
return comments.map((comment) => {
const curatedComment = {
text: comment.body,
author: comment.author,
score: comment.score,
date: comment.datetime,
};
if (comment.replies.length) {
return {
...curatedComment,
replies: curateComments(comment.replies),
};
}
return curatedComment;
});
}
function selfPostToText(item, post) {
const curatedPost = {
title: post.title,
text: post.text,
author: post.user.name,
subreddit: post.subreddit,
score: post.score,
datetime: post.datetime,
url: post.url,
comments: curateComments(item.comments),
};
return yaml.safeDump(curatedPost);
}
async function getStreams(item, post) {
if (item.self) {
return [textToStream(post.text)];
return [textToStream(selfPostToText(item, post))];
}
const sources = item.mux ? [item.url].concat(item.mux) : [item.url];
@ -97,7 +133,7 @@ async function fetchSaveUserContent(user, ep, args) {
}
async function fetchSaveDirectContent(content, host, ep) {
const data = await Promise.reduce(content.items, async (accItems, originalItem, index) => {
return Promise.reduce(content.items, async (accItems, originalItem, index) => {
const item = { ...originalItem, index };
const streams = await getStreams(item, null);

23
src/fetch/info.js
View File

@ -5,7 +5,7 @@ const Promise = require('bluebird');
const methods = require('../methods/methods.js');
const attachContentInfo = users => Promise.reduce(Object.values(users), async (accUsers, user) => ({
const attachContentInfo = (users, reddit) => Promise.reduce(Object.values(users), async (accUsers, user) => ({
...accUsers,
[user.name]: {
...user,
@ -17,14 +17,27 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a
}
try {
return [...accPosts, { ...post, content: await methods[post.host.method](post.host, post) }];
return [
...accPosts,
{
...post,
content: await methods[post.host.method](post.host, post, reddit),
},
];
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', `${error} (${post.permalink})`);
if (config.fetch.archives.preview && post.preview) {
console.log(`Found preview images for unavailable source '${post.url}' (${post.permalink})`);
return [...accPosts, { ...post, previewFallback: true, content: await methods.redditPreview(post.host, post) }];
return [
...accPosts,
{
...post,
previewFallback: true,
content: await methods.redditPreview(post.host, post),
},
];
}
return accPosts;
@ -33,8 +46,8 @@ const attachContentInfo = users => Promise.reduce(Object.values(users), async (a
},
}), {});
async function getInfo(host) {
return methods[host.method](host);
async function getInfo(host, reddit) {
return methods[host.method](host, null, reddit);
}
module.exports = {

40
src/methods/self.js
View File

@ -1,7 +1,39 @@
'use strict';
function self(host, post) {
console.log(post);
function curateComments(comments) {
return comments.map(comment => ({
id: comment.id,
url: `https://reddit.com${comment.permalink}`,
author: comment.author.name,
body: comment.body,
html: comment.body_html,
score: comment.score,
datetime: new Date(comment.created * 1000),
edited: comment.edited,
controversiality: comment.controversiality,
gilded: comment.gilded,
stickied: comment.stickied,
distinguished: comment.distinguished,
locked: comment.locked,
archived: comment.archived,
parent: comment.parent_id,
replies: comment.replies ? curateComments(comment.replies) : [],
}));
}
async function getFullPost(postId, reddit) {
return reddit
.getSubmission(postId)
.expandReplies({
limit: Infinity,
depth: Infinity,
})
.fetch();
}
async function self(host, originalPost, reddit) {
const post = await getFullPost(originalPost.id, reddit) || originalPost;
const curatedComments = curateComments(post.comments);
return {
album: null,
@ -10,11 +42,13 @@ function self(host, post) {
url: post.url,
title: post.title,
text: post.text,
author: post.author.name,
datetime: post.datetime,
comments: curatedComments,
type: 'text/plain',
self: true,
original: post,
}]
}],
};
}

4
src/save/textToStream.js
View File

@ -1,6 +1,6 @@
'use strict';
const Readable = require('stream').Readable;
const { Readable } = require('stream');
function textToStream(text) {
const stream = new Readable();
@ -9,6 +9,6 @@ function textToStream(text) {
stream.push(null);
return stream;
};
}
module.exports = textToStream;

28
src/sources/getPosts.js
View File

@ -5,7 +5,7 @@ const Promise = require('bluebird');
const getIndex = require('./getIndex.js');
const curateUser = require('../curate/user.js');
const getUser = async (username, reddit) => {
async function getUser(username, reddit) {
try {
const user = await reddit.getUser(username).fetch();
@ -22,7 +22,9 @@ const getUser = async (username, reddit) => {
const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
return Promise.reduce(postIds, (accUserPosts, postId) => Promise.resolve().then(async () => {
const post = await reddit.getSubmission(postId).fetch();
const post = await reddit
.getSubmission(postId)
.fetch();
post.direct = true;
@ -34,13 +36,31 @@ const getPostsWrap = reddit => function getPosts(postIds, userPosts = {}) {
// don't attempt to fetch deleted user
if (post.author.name === '[deleted]') {
return { ...accUserPosts, '[deleted]': { name: '[deleted]', deleted: true, posts: [post] } };
return {
...accUserPosts,
'[deleted]': {
name: '[deleted]',
deleted: true,
posts: [post],
},
};
}
const user = await getUser(post.author.name, reddit);
const { profile, posts: indexed } = await getIndex(user);
return { ...accUserPosts, [post.author.name]: { ...user, posts: [post], indexed: { profile, original: indexed, updated: [] } } };
return {
...accUserPosts,
[post.author.name]: {
...user,
posts: [post],
indexed: {
profile,
original: indexed,
updated: [],
},
},
};
}), userPosts);
};

4
src/sources/getUserPosts.js
View File

@ -25,7 +25,9 @@ async function getUser(username, reddit) {
async function getPosts(username, reddit, args) {
try {
const submissions = await reddit.getUser(username).getSubmissions({
const submissions = await reddit
.getUser(username)
.getSubmissions({
sort: args.sort,
limit: Infinity,
});