boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upgraded knex and pg versions.

This commit is contained in:
DebaucheryLibrarian 2020-09-19 00:12:15 +02:00
parent aa265fc350
commit e78bfe4c22
3 changed files with 106 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
package-lock.json generated
View File

@ -2704,9 +2704,9 @@
} }
}, },
"colorette": { "colorette": {
"version": "1.1.0", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/colorette/-/colorette-1.1.0.tgz", "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz",
"integrity": "sha512-6S062WDQUXi6hOfkO/sBPVwE5ASXY4G2+b4atvhJfSsuUUhIaUKlkjLe9692Ipyt5/a+IPF5aVTu3V5gvXq5cg==" "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw=="
}, },
"colornames": { "colornames": {
"version": "1.1.1", "version": "1.1.1",
@ -6582,32 +6582,31 @@
"integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==" "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="
}, },
"knex": { "knex": {
"version": "0.20.10", "version": "0.21.5",
"resolved": "https://registry.npmjs.org/knex/-/knex-0.20.10.tgz", "resolved": "https://registry.npmjs.org/knex/-/knex-0.21.5.tgz",
"integrity": "sha512-07D6fvY5NdvrfRPmkLLG+OrHvmAy55OX7eXkN8TMiOOI5lWJh1dC2zKjeEQJqUILMOsTnZCGqTKGaRm4t1E9xg==", "integrity": "sha512-cQj7F2D/fu03eTr6ZzYCYKdB9w7fPYlvTiU/f2OeXay52Pq5PwD+NAkcf40WDnppt/4/4KukROwlMOaE7WArcA==",
"requires": { "requires": {
"bluebird": "^3.7.2", "colorette": "1.2.1",
"colorette": "1.1.0", "commander": "^5.1.0",
"commander": "^4.1.1",
"debug": "4.1.1", "debug": "4.1.1",
"esm": "^3.2.25", "esm": "^3.2.25",
"getopts": "2.2.5", "getopts": "2.2.5",
"inherits": "~2.0.4", "inherits": "~2.0.4",
"interpret": "^2.0.0", "interpret": "^2.2.0",
"liftoff": "3.1.0", "liftoff": "3.1.0",
"lodash": "^4.17.15", "lodash": "^4.17.20",
"mkdirp": "^0.5.1", "mkdirp": "^1.0.4",
"pg-connection-string": "2.1.0", "pg-connection-string": "2.3.0",
"tarn": "^2.0.0", "tarn": "^3.0.0",
"tildify": "2.0.0", "tildify": "2.0.0",
"uuid": "^3.4.0", "uuid": "^7.0.3",
"v8flags": "^3.1.3" "v8flags": "^3.2.0"
}, },
"dependencies": { "dependencies": {
"commander": { "commander": {
"version": "4.1.1", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
"integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==" "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="
}, },
"debug": { "debug": {
"version": "4.1.1", "version": "4.1.1",
@ -6618,19 +6617,29 @@
} }
}, },
"interpret": { "interpret": {
"version": "2.0.0", "version": "2.2.0",
"resolved": "https://registry.npmjs.org/interpret/-/interpret-2.0.0.tgz", "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz",
"integrity": "sha512-e0/LknJ8wpMMhTiWcjivB+ESwIuvHnBSlBbmP/pSb8CQJldoj1p2qv7xGZ/+BtbTziYRFSz8OsvdbiX45LtYQA==" "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw=="
},
"lodash": {
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
}, },
"pg-connection-string": { "pg-connection-string": {
"version": "2.1.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.1.0.tgz", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz",
"integrity": "sha512-bhlV7Eq09JrRIvo1eKngpwuqKtJnNhZdpdOlvrPrA4dxqXPjxSrbNrfnIDmTpwMyRszrcV4kU5ZA4mMsQUrjdg==" "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w=="
}, },
"uuid": { "uuid": {
"version": "3.4.0", "version": "7.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
} }
} }
}, },
@ -8215,24 +8224,24 @@
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
}, },
"pg": { "pg": {
"version": "7.18.2", "version": "8.3.3",
"resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", "resolved": "https://registry.npmjs.org/pg/-/pg-8.3.3.tgz",
"integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", "integrity": "sha512-wmUyoQM/Xzmo62wgOdQAn5tl7u+IA1ZYK7qbuppi+3E+Gj4hlUxVHjInulieWrd0SfHi/ADriTb5ILJ/lsJrSg==",
"requires": { "requires": {
"buffer-writer": "2.0.0", "buffer-writer": "2.0.0",
"packet-reader": "1.0.0", "packet-reader": "1.0.0",
"pg-connection-string": "0.1.3", "pg-connection-string": "^2.3.0",
"pg-packet-stream": "^1.1.0", "pg-pool": "^3.2.1",
"pg-pool": "^2.0.10", "pg-protocol": "^1.2.5",
"pg-types": "^2.1.0", "pg-types": "^2.1.0",
"pgpass": "1.x", "pgpass": "1.x",
"semver": "4.3.2" "semver": "4.3.2"
}, },
"dependencies": { "dependencies": {
"pg-connection-string": { "pg-connection-string": {
"version": "0.1.3", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz",
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w=="
}, },
"semver": { "semver": {
"version": "4.3.2", "version": "4.3.2",
@ -8257,9 +8266,14 @@
"integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg=="
}, },
"pg-pool": { "pg-pool": {
"version": "2.0.10", "version": "3.2.1",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz",
"integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==" "integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA=="
},
"pg-protocol": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz",
"integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg=="
}, },
"pg-sql2": { "pg-sql2": {
"version": "4.5.0", "version": "4.5.0",
@ -8510,6 +8524,38 @@
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
}, },
"pg": {
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz",
"integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==",
"requires": {
"buffer-writer": "2.0.0",
"packet-reader": "1.0.0",
"pg-connection-string": "0.1.3",
"pg-packet-stream": "^1.1.0",
"pg-pool": "^2.0.10",
"pg-types": "^2.1.0",
"pgpass": "1.x",
"semver": "4.3.2"
},
"dependencies": {
"pg-connection-string": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
}
}
},
"pg-pool": {
"version": "2.0.10",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz",
"integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg=="
},
"semver": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
"integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c="
},
"ws": { "ws": {
"version": "6.2.1", "version": "6.2.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
@ -8545,9 +8591,9 @@
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU="
}, },
"postgres-date": { "postgres-date": {
"version": "1.0.4", "version": "1.0.7",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
"integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="
}, },
"postgres-interval": { "postgres-interval": {
"version": "1.2.0", "version": "1.2.0",
@ -10704,9 +10750,9 @@
} }
}, },
"tarn": { "tarn": {
"version": "2.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/tarn/-/tarn-2.0.0.tgz", "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.0.tgz",
"integrity": "sha512-7rNMCZd3s9bhQh47ksAQd92ADFcJUjjbyOvyFjNLwTPpGieFHMC84S+LOzw0fx1uh6hnDz/19r8CPMnIjJlMMA==" "integrity": "sha512-PKUnlDFODZueoA8owLehl8vLcgtA8u4dRuVbZc92tspDYZixjJL6TqYOmryf/PfP/EBX+2rgNcrj96NO+RPkdQ=="
}, },
"template-format": { "template-format": {
"version": "1.2.5", "version": "1.2.5",
@ -11383,9 +11429,9 @@
"dev": true "dev": true
}, },
"v8flags": { "v8flags": {
"version": "3.1.3", "version": "3.2.0",
"resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.3.tgz", "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.2.0.tgz",
"integrity": "sha512-amh9CCg3ZxkzQ48Mhcb8iX7xpAfYJgePHxWMQCBWECpOSqJUXgY26ncA61UTV0BkPqfhcy6mzwCIoP4ygxpW8w==", "integrity": "sha512-mH8etigqMfiGWdeXpaaqGfs6BndypxusHHcv2qSHyZkGEznCd/qAXCWWRzeowtL54147cktFOC4P5y+kl8d8Jg==",
"requires": { "requires": {
"homedir-polyfill": "^1.0.1" "homedir-polyfill": "^1.0.1"
} }

4
package.json
View File

@ -96,14 +96,14 @@
"graphile-utils": "^4.5.6", "graphile-utils": "^4.5.6",
"iconv-lite": "^0.5.1", "iconv-lite": "^0.5.1",
"jsdom": "^16.3.0", "jsdom": "^16.3.0",
"knex": "^0.20.10", "knex": "^0.21.5",
"knex-migrate": "^1.7.4", "knex-migrate": "^1.7.4",
"longjohn": "^0.2.12", "longjohn": "^0.2.12",
"mime": "^2.4.4", "mime": "^2.4.4",
"moment": "^2.24.0", "moment": "^2.24.0",
"nanoid": "^2.1.11", "nanoid": "^2.1.11",
"opn": "^5.5.0", "opn": "^5.5.0",
"pg": "^7.18.1", "pg": "^8.3.3",
"postgraphile": "^4.5.5", "postgraphile": "^4.5.5",
"postgraphile-plugin-connection-filter": "^1.1.3", "postgraphile-plugin-connection-filter": "^1.1.3",
"promise-task-queue": "^1.2.0", "promise-task-queue": "^1.2.0",

14
src/updates.js
View File

@ -43,8 +43,8 @@ async function filterUniqueReleases(latestReleases, accReleases) {
return { uniqueReleases, duplicateReleases }; return { uniqueReleases, duplicateReleases };
} }
function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) { function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, upcoming) {
if (releases.length === 0) { if (releasesOnPage.length === 0) {
return false; return false;
} }
@ -53,15 +53,19 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin
} }
if (argv.last) { if (argv.last) {
return releases.length > 0 && totalReleases + releases.length < argv.last; // this will keep paginating until the second condition is met on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
return releasesOnPage.length > 0 && totalReleases + releasesOnPage.length < argv.last;
} }
if (!hasDates) { if (!hasDates) {
return totalReleases + releases.length < argv.nullDateLimit; return totalReleases + releasesOnPage.length < argv.nullDateLimit;
} }
if (argv.after) { if (argv.after) {
const oldestReleaseOnPage = releases // this will keep paginating infinitely on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
const oldestReleaseOnPage = releasesOnPage
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) .sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0]; .slice(-1)[0];