From 77ec2d37474b9aa059b9d00a7d75a37f21fd3156 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sun, 27 Dec 2020 23:43:08 +0100 Subject: [PATCH] Using advanced merge library to maximize scraped data. --- package-lock.json | 68 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/deep.js | 4 +-- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d9a45ef..a761b3aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1889,6 +1889,11 @@ } } }, + "append-type": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/append-type/-/append-type-1.0.2.tgz", + "integrity": "sha512-hac740vT/SAbrFBLgLIWZqVT5PUAcGTWS5UkDDhr+OCizZSw90WKw6sWAEgGaYd2viIblggypMXwpjzHXOvAQg==" + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -1964,6 +1969,22 @@ "is-string": "^1.0.5" } }, + "array-includes-all": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/array-includes-all/-/array-includes-all-2.0.0.tgz", + "integrity": "sha512-adtLgp4gowTJojppKRsyuco1B0YK1eOa2prwr1rCpF2ofOcuclx60W3GRqUpTVi3IorufS5uIZIilHECdl/5uA==", + "requires": { + "append-type": "^1.0.1" + } + }, + "array-includes-with-glob": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/array-includes-with-glob/-/array-includes-with-glob-2.13.0.tgz", + "integrity": "sha512-J8t9J4b24wUDVtqH3GGI/HwpkrdG4HG/w/F2Q+uvuvlhDXDhW6Ymv4jWN9VF/H8FOLRTvze0fEFsY8YGFL0mAg==", + "requires": { + "matcher": "^3.0.0" + } + }, "array-slice": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz", @@ -1984,6 +2005,11 @@ "es-abstract": "^1.17.0-next.1" } }, + "arrayiffy-if-string": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/arrayiffy-if-string/-/arrayiffy-if-string-3.12.0.tgz", + "integrity": "sha512-NksgY9FtVGdU9lndRT3EiH+OuZC/rX4kxISBveLKf3TQIemz5w3Ai/s0TBIuRyaxCMCOIulBPxSKe/F2CyBybA==" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -7137,6 +7163,11 @@ "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, "logform": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz", @@ -7245,6 +7276,21 @@ "object-visit": "^1.0.0" } }, + "matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "requires": { + "escape-string-regexp": "^4.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" + } + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -8033,6 +8079,20 @@ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, + "object-merge-advanced": { + "version": "10.12.1", + "resolved": "https://registry.npmjs.org/object-merge-advanced/-/object-merge-advanced-10.12.1.tgz", + "integrity": "sha512-3K6oKbZCfxa8lJ+2jc8K0kQLeZiA4OWF9ScoTsq3TwaeIfjsn9sn70iaY7P/l/glCaO6nD8C8nyvVI/zZIcu9w==", + "requires": { + "array-includes-all": "^2.0.0", + "array-includes-with-glob": "^2.13.0", + "arrayiffy-if-string": "^3.12.0", + "lodash.clonedeep": "^4.5.0", + "lodash.includes": "^4.3.0", + "lodash.uniq": "^4.5.0", + "util-nonempty": "^2.10.0" + } + }, "object-visit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", @@ -11515,6 +11575,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "util-nonempty": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/util-nonempty/-/util-nonempty-2.10.0.tgz", + "integrity": "sha512-kidgZTwmgqhxIUAY71KQDJCQ10NcX+yF2sjata4D831ONPUMEtg2Q5vhYVF+2Lx4RVnJKMhcu2zRz8/7Fv7+4g==", + "requires": { + "lodash.isplainobject": "^4.0.6" + } + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/package.json b/package.json index bd71961f..89742113 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "mitt": "^2.1.0", "moment": "^2.24.0", "nanoid": "^2.1.11", + "object-merge-advanced": "^10.12.1", "opn": "^5.5.0", "pg": "^8.5.1", "postgraphile": "^4.10.0", diff --git a/src/deep.js b/src/deep.js index 77ecfdda..c625d730 100644 --- a/src/deep.js +++ b/src/deep.js @@ -1,6 +1,7 @@ 'use strict'; const Promise = require('bluebird'); +const merge = require('object-merge-advanced'); const argv = require('./argv'); const include = require('./utils/argv-include')(argv); @@ -122,8 +123,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') { : await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null); const mergedRelease = { - ...baseRelease, - ...scrapedRelease, + ...merge(baseRelease, scrapedRelease), deep: !!scrapedRelease, entity, };