boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Using advanced merge library to maximize scraped data.

This commit is contained in:
DebaucheryLibrarian 2020-12-27 23:43:08 +01:00
parent 9c926a1d81
commit 77ec2d3747
3 changed files with 71 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
package-lock.json generated
View File

@ -1889,6 +1889,11 @@
}
}
},
"append-type": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/append-type/-/append-type-1.0.2.tgz",
"integrity": "sha512-hac740vT/SAbrFBLgLIWZqVT5PUAcGTWS5UkDDhr+OCizZSw90WKw6sWAEgGaYd2viIblggypMXwpjzHXOvAQg=="
},
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@ -1964,6 +1969,22 @@
"is-string": "^1.0.5"
}
},
"array-includes-all": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/array-includes-all/-/array-includes-all-2.0.0.tgz",
"integrity": "sha512-adtLgp4gowTJojppKRsyuco1B0YK1eOa2prwr1rCpF2ofOcuclx60W3GRqUpTVi3IorufS5uIZIilHECdl/5uA==",
"requires": {
"append-type": "^1.0.1"
}
},
"array-includes-with-glob": {
"version": "2.13.0",
"resolved": "https://registry.npmjs.org/array-includes-with-glob/-/array-includes-with-glob-2.13.0.tgz",
"integrity": "sha512-J8t9J4b24wUDVtqH3GGI/HwpkrdG4HG/w/F2Q+uvuvlhDXDhW6Ymv4jWN9VF/H8FOLRTvze0fEFsY8YGFL0mAg==",
"requires": {
"matcher": "^3.0.0"
}
},
"array-slice": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
@ -1984,6 +2005,11 @@
"es-abstract": "^1.17.0-next.1"
}
},
"arrayiffy-if-string": {
"version": "3.12.0",
"resolved": "https://registry.npmjs.org/arrayiffy-if-string/-/arrayiffy-if-string-3.12.0.tgz",
"integrity": "sha512-NksgY9FtVGdU9lndRT3EiH+OuZC/rX4kxISBveLKf3TQIemz5w3Ai/s0TBIuRyaxCMCOIulBPxSKe/F2CyBybA=="
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@ -7137,6 +7163,11 @@
"resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
"integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg="
},
"lodash.uniq": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
"integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
},
"logform": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz",
@ -7245,6 +7276,21 @@
"object-visit": "^1.0.0"
}
},
"matcher": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
"requires": {
"escape-string-regexp": "^4.0.0"
},
"dependencies": {
"escape-string-regexp": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
"integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="
}
}
},
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@ -8033,6 +8079,20 @@
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
},
"object-merge-advanced": {
"version": "10.12.1",
"resolved": "https://registry.npmjs.org/object-merge-advanced/-/object-merge-advanced-10.12.1.tgz",
"integrity": "sha512-3K6oKbZCfxa8lJ+2jc8K0kQLeZiA4OWF9ScoTsq3TwaeIfjsn9sn70iaY7P/l/glCaO6nD8C8nyvVI/zZIcu9w==",
"requires": {
"array-includes-all": "^2.0.0",
"array-includes-with-glob": "^2.13.0",
"arrayiffy-if-string": "^3.12.0",
"lodash.clonedeep": "^4.5.0",
"lodash.includes": "^4.3.0",
"lodash.uniq": "^4.5.0",
"util-nonempty": "^2.10.0"
}
},
"object-visit": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
@ -11515,6 +11575,14 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"util-nonempty": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/util-nonempty/-/util-nonempty-2.10.0.tgz",
"integrity": "sha512-kidgZTwmgqhxIUAY71KQDJCQ10NcX+yF2sjata4D831ONPUMEtg2Q5vhYVF+2Lx4RVnJKMhcu2zRz8/7Fv7+4g==",
"requires": {
"lodash.isplainobject": "^4.0.6"
}
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

1
package.json
View File

@ -110,6 +110,7 @@
"mitt": "^2.1.0",
"moment": "^2.24.0",
"nanoid": "^2.1.11",
"object-merge-advanced": "^10.12.1",
"opn": "^5.5.0",
"pg": "^8.5.1",
"postgraphile": "^4.10.0",

4
src/deep.js
View File

@ -1,6 +1,7 @@
'use strict';
const Promise = require('bluebird');
const merge = require('object-merge-advanced');
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
@ -122,8 +123,7 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
const mergedRelease = {
...baseRelease,
...scrapedRelease,
...merge(baseRelease, scrapedRelease),
deep: !!scrapedRelease,
entity,
};