DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Passing recursive parameters to all scraper methods. Using throttle parameters in MindGeek scraper, fixed missing slug breaking scene and actor URLs.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-02-10 03:23:48 +01:00
parent 62ad786318
commit 7ff222ce25
7 changed files with 59 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/actors.js
View File

@@ -30,6 +30,8 @@ const { deleteScenes } = require('./releases');
const slugify = require('./utils/slugify');
const capitalize = require('./utils/capitalize');
const resolvePlace = require('./utils/resolve-place');
const { resolveLayoutScraper } = require('./scrapers/resolve');
const getRecursiveParameters = require('./utils/get-recursive-parameters');
const hairColors = {
'jet-black': 'black',
@@ -637,10 +639,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
const entity = entitiesBySlug[scraperSlug] || null;
const scraper = scrapers[scraperSlug];
const layoutScraper = scraper?.[entity.parameters?.layout]
|| scraper?.[entity.parent?.parameters?.layout]
|| scraper?.[entity.parent?.parent?.parameters?.layout]
|| scraper;
const layoutScraper = resolveLayoutScraper(entity, scraper);
const context = {
...entity,
@@ -649,11 +648,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
network: entity?.parent,
entity,
scraper: scraperSlug,
parameters: {
...entity?.parent?.parent?.parameters,
...entity?.parent?.parameters,
...entity?.parameters,
},
parameters: getRecursiveParameters(entity),
};
const label = context.entity?.name;