DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Selecting included networks with infinite parent depth to facilitate scraper resolve.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-02-02 01:31:12 +01:00
parent 46c0b269c3
commit d5cdfb36a9
4 changed files with 101 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/deep.js
View File

@@ -5,10 +5,10 @@ const merge = require('object-merge-advanced');
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const { resolveScraper, resolveLayoutScraper } = require('./scrapers/resolve');
const { fetchReleaseEntities, urlToSiteSlug } = require('./entities');
const logger = require('./logger')(__filename);
const qu = require('./utils/qu');
const scrapers = require('./scrapers/scrapers');
function toBaseReleases(baseReleasesOrUrls, entity = null) {
if (!baseReleasesOrUrls) {
@@ -68,30 +68,6 @@ async function fetchScene(scraper, url, entity, baseRelease, options) {
return null;
}
function findScraper(entity) {
if (scrapers.releases[entity.slug]) {
return scrapers.releases[entity.slug];
}
if (entity.parent) {
return findScraper(entity.parent);
}
return null;
}
function findLayoutScraper(entity, scraper) {
if (scraper?.[entity.parameters?.layout]) {
return scraper[entity.parameters.layout];
}
if (entity.parent) {
return findLayoutScraper(entity.parent, scraper);
}
return scraper;
}
async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
const entity = baseRelease.entity || entitiesBySlug[urlToSiteSlug(baseRelease.url)];
@@ -107,8 +83,8 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
};
}
const scraper = findScraper(entity);
const layoutScraper = findLayoutScraper(entity, scraper);
const scraper = resolveScraper(entity);
const layoutScraper = resolveLayoutScraper(entity, scraper);
if (!layoutScraper) {
logger.warn(`Could not find scraper for ${baseRelease.url}`);