Using advanced merge library to maximize scraped data.

package-lock.json

@@ -1889,6 +1889,11 @@
                 }
             }
         },
+        "append-type": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/append-type/-/append-type-1.0.2.tgz",
+            "integrity": "sha512-hac740vT/SAbrFBLgLIWZqVT5PUAcGTWS5UkDDhr+OCizZSw90WKw6sWAEgGaYd2viIblggypMXwpjzHXOvAQg=="
+        },
         "aproba": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -1964,6 +1969,22 @@
                 "is-string": "^1.0.5"
             }
         },
+        "array-includes-all": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/array-includes-all/-/array-includes-all-2.0.0.tgz",
+            "integrity": "sha512-adtLgp4gowTJojppKRsyuco1B0YK1eOa2prwr1rCpF2ofOcuclx60W3GRqUpTVi3IorufS5uIZIilHECdl/5uA==",
+            "requires": {
+                "append-type": "^1.0.1"
+            }
+        },
+        "array-includes-with-glob": {
+            "version": "2.13.0",
+            "resolved": "https://registry.npmjs.org/array-includes-with-glob/-/array-includes-with-glob-2.13.0.tgz",
+            "integrity": "sha512-J8t9J4b24wUDVtqH3GGI/HwpkrdG4HG/w/F2Q+uvuvlhDXDhW6Ymv4jWN9VF/H8FOLRTvze0fEFsY8YGFL0mAg==",
+            "requires": {
+                "matcher": "^3.0.0"
+            }
+        },
         "array-slice": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
@@ -1984,6 +2005,11 @@
                 "es-abstract": "^1.17.0-next.1"
             }
         },
+        "arrayiffy-if-string": {
+            "version": "3.12.0",
+            "resolved": "https://registry.npmjs.org/arrayiffy-if-string/-/arrayiffy-if-string-3.12.0.tgz",
+            "integrity": "sha512-NksgY9FtVGdU9lndRT3EiH+OuZC/rX4kxISBveLKf3TQIemz5w3Ai/s0TBIuRyaxCMCOIulBPxSKe/F2CyBybA=="
+        },
         "asn1": {
             "version": "0.2.4",
             "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -7137,6 +7163,11 @@
             "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
             "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg="
         },
+        "lodash.uniq": {
+            "version": "4.5.0",
+            "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
+            "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
+        },
         "logform": {
             "version": "2.1.2",
             "resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz",
@@ -7245,6 +7276,21 @@
                 "object-visit": "^1.0.0"
             }
         },
+        "matcher": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
+            "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
+            "requires": {
+                "escape-string-regexp": "^4.0.0"
+            },
+            "dependencies": {
+                "escape-string-regexp": {
+                    "version": "4.0.0",
+                    "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+                    "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="
+                }
+            }
+        },
         "media-typer": {
             "version": "0.3.0",
             "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -8033,6 +8079,20 @@
             "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
             "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
         },
+        "object-merge-advanced": {
+            "version": "10.12.1",
+            "resolved": "https://registry.npmjs.org/object-merge-advanced/-/object-merge-advanced-10.12.1.tgz",
+            "integrity": "sha512-3K6oKbZCfxa8lJ+2jc8K0kQLeZiA4OWF9ScoTsq3TwaeIfjsn9sn70iaY7P/l/glCaO6nD8C8nyvVI/zZIcu9w==",
+            "requires": {
+                "array-includes-all": "^2.0.0",
+                "array-includes-with-glob": "^2.13.0",
+                "arrayiffy-if-string": "^3.12.0",
+                "lodash.clonedeep": "^4.5.0",
+                "lodash.includes": "^4.3.0",
+                "lodash.uniq": "^4.5.0",
+                "util-nonempty": "^2.10.0"
+            }
+        },
         "object-visit": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
@@ -11515,6 +11575,14 @@
             "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
             "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
         },
+        "util-nonempty": {
+            "version": "2.10.0",
+            "resolved": "https://registry.npmjs.org/util-nonempty/-/util-nonempty-2.10.0.tgz",
+            "integrity": "sha512-kidgZTwmgqhxIUAY71KQDJCQ10NcX+yF2sjata4D831ONPUMEtg2Q5vhYVF+2Lx4RVnJKMhcu2zRz8/7Fv7+4g==",
+            "requires": {
+                "lodash.isplainobject": "^4.0.6"
+            }
+        },
         "utils-merge": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

package.json

@@ -110,6 +110,7 @@
         "mitt": "^2.1.0",
         "moment": "^2.24.0",
         "nanoid": "^2.1.11",
+        "object-merge-advanced": "^10.12.1",
         "opn": "^5.5.0",
         "pg": "^8.5.1",
         "postgraphile": "^4.10.0",

src/deep.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const Promise = require('bluebird');
+const merge = require('object-merge-advanced');
 
 const argv = require('./argv');
 const include = require('./utils/argv-include')(argv);
@@ -122,8 +123,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
 			: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
 
 		const mergedRelease = {
-			...baseRelease,
+			...merge(baseRelease, scrapedRelease),
-			...scrapedRelease,
 			deep: !!scrapedRelease,
 			entity,
 		};