Upgraded knex and pg versions.
This commit is contained in:
DebaucheryLibrarian
parent
aa265fc350
commit
e78bfe4c22
|
|
@ -2704,9 +2704,9 @@
|
|||
}
|
||||
},
|
||||
"colorette": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/colorette/-/colorette-1.1.0.tgz",
|
||||
"integrity": "sha512-6S062WDQUXi6hOfkO/sBPVwE5ASXY4G2+b4atvhJfSsuUUhIaUKlkjLe9692Ipyt5/a+IPF5aVTu3V5gvXq5cg=="
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz",
|
||||
"integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw=="
|
||||
},
|
||||
"colornames": {
|
||||
"version": "1.1.1",
|
||||
|
|
@ -6582,32 +6582,31 @@
|
|||
"integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="
|
||||
},
|
||||
"knex": {
|
||||
"version": "0.20.10",
|
||||
"resolved": "https://registry.npmjs.org/knex/-/knex-0.20.10.tgz",
|
||||
"integrity": "sha512-07D6fvY5NdvrfRPmkLLG+OrHvmAy55OX7eXkN8TMiOOI5lWJh1dC2zKjeEQJqUILMOsTnZCGqTKGaRm4t1E9xg==",
|
||||
"version": "0.21.5",
|
||||
"resolved": "https://registry.npmjs.org/knex/-/knex-0.21.5.tgz",
|
||||
"integrity": "sha512-cQj7F2D/fu03eTr6ZzYCYKdB9w7fPYlvTiU/f2OeXay52Pq5PwD+NAkcf40WDnppt/4/4KukROwlMOaE7WArcA==",
|
||||
"requires": {
|
||||
"bluebird": "^3.7.2",
|
||||
"colorette": "1.1.0",
|
||||
"commander": "^4.1.1",
|
||||
"colorette": "1.2.1",
|
||||
"commander": "^5.1.0",
|
||||
"debug": "4.1.1",
|
||||
"esm": "^3.2.25",
|
||||
"getopts": "2.2.5",
|
||||
"inherits": "~2.0.4",
|
||||
"interpret": "^2.0.0",
|
||||
"interpret": "^2.2.0",
|
||||
"liftoff": "3.1.0",
|
||||
"lodash": "^4.17.15",
|
||||
"mkdirp": "^0.5.1",
|
||||
"pg-connection-string": "2.1.0",
|
||||
"tarn": "^2.0.0",
|
||||
"lodash": "^4.17.20",
|
||||
"mkdirp": "^1.0.4",
|
||||
"pg-connection-string": "2.3.0",
|
||||
"tarn": "^3.0.0",
|
||||
"tildify": "2.0.0",
|
||||
"uuid": "^3.4.0",
|
||||
"v8flags": "^3.1.3"
|
||||
"uuid": "^7.0.3",
|
||||
"v8flags": "^3.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"commander": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
|
||||
"integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
|
||||
"integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg=="
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
|
|
@ -6618,19 +6617,29 @@
|
|||
}
|
||||
},
|
||||
"interpret": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/interpret/-/interpret-2.0.0.tgz",
|
||||
"integrity": "sha512-e0/LknJ8wpMMhTiWcjivB+ESwIuvHnBSlBbmP/pSb8CQJldoj1p2qv7xGZ/+BtbTziYRFSz8OsvdbiX45LtYQA=="
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz",
|
||||
"integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw=="
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.20",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
|
||||
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
|
||||
},
|
||||
"mkdirp": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
|
||||
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
|
||||
},
|
||||
"pg-connection-string": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.1.0.tgz",
|
||||
"integrity": "sha512-bhlV7Eq09JrRIvo1eKngpwuqKtJnNhZdpdOlvrPrA4dxqXPjxSrbNrfnIDmTpwMyRszrcV4kU5ZA4mMsQUrjdg=="
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz",
|
||||
"integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w=="
|
||||
},
|
||||
"uuid": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
|
||||
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
|
||||
"integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -8215,24 +8224,24 @@
|
|||
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
|
||||
},
|
||||
"pg": {
|
||||
"version": "7.18.2",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz",
|
||||
"integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==",
|
||||
"version": "8.3.3",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.3.3.tgz",
|
||||
"integrity": "sha512-wmUyoQM/Xzmo62wgOdQAn5tl7u+IA1ZYK7qbuppi+3E+Gj4hlUxVHjInulieWrd0SfHi/ADriTb5ILJ/lsJrSg==",
|
||||
"requires": {
|
||||
"buffer-writer": "2.0.0",
|
||||
"packet-reader": "1.0.0",
|
||||
"pg-connection-string": "0.1.3",
|
||||
"pg-packet-stream": "^1.1.0",
|
||||
"pg-pool": "^2.0.10",
|
||||
"pg-connection-string": "^2.3.0",
|
||||
"pg-pool": "^3.2.1",
|
||||
"pg-protocol": "^1.2.5",
|
||||
"pg-types": "^2.1.0",
|
||||
"pgpass": "1.x",
|
||||
"semver": "4.3.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"pg-connection-string": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
|
||||
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz",
|
||||
"integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w=="
|
||||
},
|
||||
"semver": {
|
||||
"version": "4.3.2",
|
||||
|
|
@ -8257,9 +8266,14 @@
|
|||
"integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg=="
|
||||
},
|
||||
"pg-pool": {
|
||||
"version": "2.0.10",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz",
|
||||
"integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg=="
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz",
|
||||
"integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA=="
|
||||
},
|
||||
"pg-protocol": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz",
|
||||
"integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg=="
|
||||
},
|
||||
"pg-sql2": {
|
||||
"version": "4.5.0",
|
||||
|
|
@ -8510,6 +8524,38 @@
|
|||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
|
||||
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
|
||||
},
|
||||
"pg": {
|
||||
"version": "7.18.2",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz",
|
||||
"integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==",
|
||||
"requires": {
|
||||
"buffer-writer": "2.0.0",
|
||||
"packet-reader": "1.0.0",
|
||||
"pg-connection-string": "0.1.3",
|
||||
"pg-packet-stream": "^1.1.0",
|
||||
"pg-pool": "^2.0.10",
|
||||
"pg-types": "^2.1.0",
|
||||
"pgpass": "1.x",
|
||||
"semver": "4.3.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"pg-connection-string": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
|
||||
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
|
||||
}
|
||||
}
|
||||
},
|
||||
"pg-pool": {
|
||||
"version": "2.0.10",
|
||||
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz",
|
||||
"integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg=="
|
||||
},
|
||||
"semver": {
|
||||
"version": "4.3.2",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
|
||||
"integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c="
|
||||
},
|
||||
"ws": {
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
|
||||
|
|
@ -8545,9 +8591,9 @@
|
|||
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU="
|
||||
},
|
||||
"postgres-date": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz",
|
||||
"integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA=="
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
|
||||
"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="
|
||||
},
|
||||
"postgres-interval": {
|
||||
"version": "1.2.0",
|
||||
|
|
@ -10704,9 +10750,9 @@
|
|||
}
|
||||
},
|
||||
"tarn": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tarn/-/tarn-2.0.0.tgz",
|
||||
"integrity": "sha512-7rNMCZd3s9bhQh47ksAQd92ADFcJUjjbyOvyFjNLwTPpGieFHMC84S+LOzw0fx1uh6hnDz/19r8CPMnIjJlMMA=="
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.0.tgz",
|
||||
"integrity": "sha512-PKUnlDFODZueoA8owLehl8vLcgtA8u4dRuVbZc92tspDYZixjJL6TqYOmryf/PfP/EBX+2rgNcrj96NO+RPkdQ=="
|
||||
},
|
||||
"template-format": {
|
||||
"version": "1.2.5",
|
||||
|
|
@ -11383,9 +11429,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"v8flags": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.3.tgz",
|
||||
"integrity": "sha512-amh9CCg3ZxkzQ48Mhcb8iX7xpAfYJgePHxWMQCBWECpOSqJUXgY26ncA61UTV0BkPqfhcy6mzwCIoP4ygxpW8w==",
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.2.0.tgz",
|
||||
"integrity": "sha512-mH8etigqMfiGWdeXpaaqGfs6BndypxusHHcv2qSHyZkGEznCd/qAXCWWRzeowtL54147cktFOC4P5y+kl8d8Jg==",
|
||||
"requires": {
|
||||
"homedir-polyfill": "^1.0.1"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,14 +96,14 @@
|
|||
"graphile-utils": "^4.5.6",
|
||||
"iconv-lite": "^0.5.1",
|
||||
"jsdom": "^16.3.0",
|
||||
"knex": "^0.20.10",
|
||||
"knex": "^0.21.5",
|
||||
"knex-migrate": "^1.7.4",
|
||||
"longjohn": "^0.2.12",
|
||||
"mime": "^2.4.4",
|
||||
"moment": "^2.24.0",
|
||||
"nanoid": "^2.1.11",
|
||||
"opn": "^5.5.0",
|
||||
"pg": "^7.18.1",
|
||||
"pg": "^8.3.3",
|
||||
"postgraphile": "^4.5.5",
|
||||
"postgraphile-plugin-connection-filter": "^1.1.3",
|
||||
"promise-task-queue": "^1.2.0",
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@ async function filterUniqueReleases(latestReleases, accReleases) {
|
|||
return { uniqueReleases, duplicateReleases };
|
||||
}
|
||||
|
||||
function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) {
|
||||
if (releases.length === 0) {
|
||||
function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, upcoming) {
|
||||
if (releasesOnPage.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -53,15 +53,19 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin
|
|||
}
|
||||
|
||||
if (argv.last) {
|
||||
return releases.length > 0 && totalReleases + releases.length < argv.last;
|
||||
// this will keep paginating until the second condition is met on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
|
||||
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
|
||||
return releasesOnPage.length > 0 && totalReleases + releasesOnPage.length < argv.last;
|
||||
}
|
||||
|
||||
if (!hasDates) {
|
||||
return totalReleases + releases.length < argv.nullDateLimit;
|
||||
return totalReleases + releasesOnPage.length < argv.nullDateLimit;
|
||||
}
|
||||
|
||||
if (argv.after) {
|
||||
const oldestReleaseOnPage = releases
|
||||
// this will keep paginating infinitely on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020)
|
||||
// checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier
|
||||
const oldestReleaseOnPage = releasesOnPage
|
||||
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
|
||||
.slice(-1)[0];
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue