boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Passing resolved parameters into before-scrape methods.

This commit is contained in:
DebaucheryLibrarian 2022-02-25 22:13:41 +01:00
parent a5afffc968
commit 5deba6b90f
3 changed files with 4 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/default.js
View File

@ -298,18 +298,6 @@ module.exports = {
interval: 1000, interval: 1000,
concurrency: 1, concurrency: 1,
}, },
'www.realitykings.com': {
interval: 1000,
concurrency: 1,
},
'www.mofos.com': {
interval: 1000,
concurrency: 1,
},
'www.twistys.com': {
interval: 1000,
concurrency: 1,
},
'westcoastproductions.com': { 'westcoastproductions.com': {
interval: 100, interval: 100,
concurrency: 1, concurrency: 1,

3
src/deep.js
View File

@ -200,7 +200,8 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
async function scrapeReleases(baseReleases, entitiesBySlug, type) { async function scrapeReleases(baseReleases, entitiesBySlug, type) {
const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesBySlug).map(async ([slug, entity]) => { const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesBySlug).map(async ([slug, entity]) => {
if (entity.scraper?.beforeFetchScenes) { if (entity.scraper?.beforeFetchScenes) {
const preData = await entity.scraper.beforeFetchScenes(entity); const parameters = getRecursiveParameters(entity);
const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
return [slug, { ...entity, preData }]; return [slug, { ...entity, preData }];
} }

3
src/updates.js
View File

@ -269,7 +269,8 @@ async function scrapeNetworkSequential(networkEntity) {
async function getBeforeNetwork(networkEntity) { async function getBeforeNetwork(networkEntity) {
try { try {
return await networkEntity.scraper?.beforeNetwork?.(networkEntity); const parameters = getRecursiveParameters(networkEntity);
return await networkEntity.scraper?.beforeNetwork?.(networkEntity, parameters);
} catch (error) { } catch (error) {
if (networkEntity.scraper?.requireBeforeNetwork === false) { if (networkEntity.scraper?.requireBeforeNetwork === false) {
return null; return null;