Passing recursive parameters to all scraper methods. Using throttle parameters in MindGeek scraper, fixed missing slug breaking scene and actor URLs.

This commit is contained in:
DebaucheryLibrarian
2021-02-10 03:23:48 +01:00
parent 62ad786318
commit 7ff222ce25
7 changed files with 59 additions and 26 deletions

View File

@@ -11,6 +11,7 @@ const { curateRelease } = require('./releases');
const include = require('./utils/argv-include')(argv);
const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
const { fetchIncludedEntities } = require('./entities');
const getRecursiveParameters = require('./utils/get-recursive-parameters');
const emptyReleases = { uniqueReleases: [], duplicateReleases: [] };
@@ -97,6 +98,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const options = {
...config.options[scraper.slug],
...include,
parameters: getRecursiveParameters(entity),
};
const pageReleases = isUpcoming