DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Passing resolved parameters into before-scrape methods.

This commit is contained in:
DebaucheryLibrarian 2022-02-25 22:13:41 +01:00
parent a5afffc968
commit 5deba6b90f
3 changed files with 4 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/default.js
View File

@ -298,18 +298,6 @@ module.exports = {
interval: 1000,
concurrency: 1,
},
'www.realitykings.com': {
interval: 1000,
concurrency: 1,
},
'www.mofos.com': {
interval: 1000,
concurrency: 1,
},
'www.twistys.com': {
interval: 1000,
concurrency: 1,
},
'westcoastproductions.com': {
interval: 100,
concurrency: 1,

3
src/deep.js
View File

@ -200,7 +200,8 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
async function scrapeReleases(baseReleases, entitiesBySlug, type) {
const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesBySlug).map(async ([slug, entity]) => {
if (entity.scraper?.beforeFetchScenes) {
const preData = await entity.scraper.beforeFetchScenes(entity);
const parameters = getRecursiveParameters(entity);
const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
return [slug, { ...entity, preData }];
}

3
src/updates.js
View File

@ -269,7 +269,8 @@ async function scrapeNetworkSequential(networkEntity) {
async function getBeforeNetwork(networkEntity) {
try {
return await networkEntity.scraper?.beforeNetwork?.(networkEntity);
const parameters = getRecursiveParameters(networkEntity);
return await networkEntity.scraper?.beforeNetwork?.(networkEntity, parameters);
} catch (error) {
if (networkEntity.scraper?.requireBeforeNetwork === false) {
return null;