diff --git a/package-lock.json b/package-lock.json index 40ebc423..ce4da19a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2704,9 +2704,9 @@ } }, "colorette": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.1.0.tgz", - "integrity": "sha512-6S062WDQUXi6hOfkO/sBPVwE5ASXY4G2+b4atvhJfSsuUUhIaUKlkjLe9692Ipyt5/a+IPF5aVTu3V5gvXq5cg==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", + "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==" }, "colornames": { "version": "1.1.1", @@ -6582,32 +6582,31 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==" }, "knex": { - "version": "0.20.10", - "resolved": "https://registry.npmjs.org/knex/-/knex-0.20.10.tgz", - "integrity": "sha512-07D6fvY5NdvrfRPmkLLG+OrHvmAy55OX7eXkN8TMiOOI5lWJh1dC2zKjeEQJqUILMOsTnZCGqTKGaRm4t1E9xg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/knex/-/knex-0.21.5.tgz", + "integrity": "sha512-cQj7F2D/fu03eTr6ZzYCYKdB9w7fPYlvTiU/f2OeXay52Pq5PwD+NAkcf40WDnppt/4/4KukROwlMOaE7WArcA==", "requires": { - "bluebird": "^3.7.2", - "colorette": "1.1.0", - "commander": "^4.1.1", + "colorette": "1.2.1", + "commander": "^5.1.0", "debug": "4.1.1", "esm": "^3.2.25", "getopts": "2.2.5", "inherits": "~2.0.4", - "interpret": "^2.0.0", + "interpret": "^2.2.0", "liftoff": "3.1.0", - "lodash": "^4.17.15", - "mkdirp": "^0.5.1", - "pg-connection-string": "2.1.0", - "tarn": "^2.0.0", + "lodash": "^4.17.20", + "mkdirp": "^1.0.4", + "pg-connection-string": "2.3.0", + "tarn": "^3.0.0", "tildify": "2.0.0", - "uuid": "^3.4.0", - "v8flags": "^3.1.3" + "uuid": "^7.0.3", + "v8flags": "^3.2.0" }, "dependencies": { "commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==" + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" }, "debug": { "version": "4.1.1", @@ -6618,19 +6617,29 @@ } }, "interpret": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.0.0.tgz", - "integrity": "sha512-e0/LknJ8wpMMhTiWcjivB+ESwIuvHnBSlBbmP/pSb8CQJldoj1p2qv7xGZ/+BtbTziYRFSz8OsvdbiX45LtYQA==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" }, "pg-connection-string": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.1.0.tgz", - "integrity": "sha512-bhlV7Eq09JrRIvo1eKngpwuqKtJnNhZdpdOlvrPrA4dxqXPjxSrbNrfnIDmTpwMyRszrcV4kU5ZA4mMsQUrjdg==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz", + "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w==" }, "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", + "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" } } }, @@ -8215,24 +8224,24 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "pg": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", - "integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", + "version": "8.3.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.3.3.tgz", + "integrity": "sha512-wmUyoQM/Xzmo62wgOdQAn5tl7u+IA1ZYK7qbuppi+3E+Gj4hlUxVHjInulieWrd0SfHi/ADriTb5ILJ/lsJrSg==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", - "pg-connection-string": "0.1.3", - "pg-packet-stream": "^1.1.0", - "pg-pool": "^2.0.10", + "pg-connection-string": "^2.3.0", + "pg-pool": "^3.2.1", + "pg-protocol": "^1.2.5", "pg-types": "^2.1.0", "pgpass": "1.x", "semver": "4.3.2" }, "dependencies": { "pg-connection-string": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", - "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz", + "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w==" }, "semver": { "version": "4.3.2", @@ -8257,9 +8266,14 @@ "integrity": "sha512-kRBH0tDIW/8lfnnOyTwKD23ygJ/kexQVXZs7gEyBljw4FYqimZFxnMMx50ndZ8In77QgfGuItS5LLclC2TtjYg==" }, "pg-pool": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", - "integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==" + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz", + "integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA==" + }, + "pg-protocol": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz", + "integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg==" }, "pg-sql2": { "version": "4.5.0", @@ -8510,6 +8524,38 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, + "pg": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.18.2.tgz", + "integrity": "sha512-Mvt0dGYMwvEADNKy5PMQGlzPudKcKKzJds/VbOeZJpb6f/pI3mmoXX0JksPgI3l3JPP/2Apq7F36O63J7mgveA==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-packet-stream": "^1.1.0", + "pg-pool": "^2.0.10", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "dependencies": { + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + } + } + }, + "pg-pool": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.10.tgz", + "integrity": "sha512-qdwzY92bHf3nwzIUcj+zJ0Qo5lpG/YxchahxIN8+ZVmXqkahKXsnl2aiJPHLYN9o5mB/leG+Xh6XKxtP7e0sjg==" + }, + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + }, "ws": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", @@ -8545,9 +8591,9 @@ "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" }, "postgres-date": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", - "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" }, "postgres-interval": { "version": "1.2.0", @@ -10704,9 +10750,9 @@ } }, "tarn": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tarn/-/tarn-2.0.0.tgz", - "integrity": "sha512-7rNMCZd3s9bhQh47ksAQd92ADFcJUjjbyOvyFjNLwTPpGieFHMC84S+LOzw0fx1uh6hnDz/19r8CPMnIjJlMMA==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.0.tgz", + "integrity": "sha512-PKUnlDFODZueoA8owLehl8vLcgtA8u4dRuVbZc92tspDYZixjJL6TqYOmryf/PfP/EBX+2rgNcrj96NO+RPkdQ==" }, "template-format": { "version": "1.2.5", @@ -11383,9 +11429,9 @@ "dev": true }, "v8flags": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.1.3.tgz", - "integrity": "sha512-amh9CCg3ZxkzQ48Mhcb8iX7xpAfYJgePHxWMQCBWECpOSqJUXgY26ncA61UTV0BkPqfhcy6mzwCIoP4ygxpW8w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-3.2.0.tgz", + "integrity": "sha512-mH8etigqMfiGWdeXpaaqGfs6BndypxusHHcv2qSHyZkGEznCd/qAXCWWRzeowtL54147cktFOC4P5y+kl8d8Jg==", "requires": { "homedir-polyfill": "^1.0.1" } diff --git a/package.json b/package.json index 2a1ee315..c66beb49 100644 --- a/package.json +++ b/package.json @@ -96,14 +96,14 @@ "graphile-utils": "^4.5.6", "iconv-lite": "^0.5.1", "jsdom": "^16.3.0", - "knex": "^0.20.10", + "knex": "^0.21.5", "knex-migrate": "^1.7.4", "longjohn": "^0.2.12", "mime": "^2.4.4", "moment": "^2.24.0", "nanoid": "^2.1.11", "opn": "^5.5.0", - "pg": "^7.18.1", + "pg": "^8.3.3", "postgraphile": "^4.5.5", "postgraphile-plugin-connection-filter": "^1.1.3", "promise-task-queue": "^1.2.0", diff --git a/src/updates.js b/src/updates.js index 9bca7fc3..060d6f58 100644 --- a/src/updates.js +++ b/src/updates.js @@ -43,8 +43,8 @@ async function filterUniqueReleases(latestReleases, accReleases) { return { uniqueReleases, duplicateReleases }; } -function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcoming) { - if (releases.length === 0) { +function needNextPage(releasesOnPage, uniqueReleases, totalReleases, hasDates, upcoming) { + if (releasesOnPage.length === 0) { return false; } @@ -53,15 +53,19 @@ function needNextPage(releases, uniqueReleases, totalReleases, hasDates, upcomin } if (argv.last) { - return releases.length > 0 && totalReleases + releases.length < argv.last; + // this will keep paginating until the second condition is met on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020) + // checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier + return releasesOnPage.length > 0 && totalReleases + releasesOnPage.length < argv.last; } if (!hasDates) { - return totalReleases + releases.length < argv.nullDateLimit; + return totalReleases + releasesOnPage.length < argv.nullDateLimit; } if (argv.after) { - const oldestReleaseOnPage = releases + // this will keep paginating infinitely on sites that will keep serving the last page if you exceed the last page number (e.g. HardX as of september 2020) + // checking unqiueReleases > 0 could prevent that, but this would stop pagination prematurely if we already scraped a full page of data earlier + const oldestReleaseOnPage = releasesOnPage .sort((releaseA, releaseB) => releaseB.date - releaseA.date) .slice(-1)[0];