DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Fixed beforeFetchScenes not running for update releases. Fixed Hookup Hotshot deep scrape broken due missing NATS cookie.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-10-03 06:32:03 +02:00
parent b700e05c5e
commit 84ac366e36
5 changed files with 84 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/deep.js
+31 -10
View File
@@ -123,15 +123,21 @@ function fetchMovie(scraper, url, entity, baseRelease, options) {
return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
}
async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
const entity = baseRelease.entity || entitiesByHostname[urlToHostname(baseRelease.url)];
function shouldDeepScrape(baseRelease) {
return argv.deep && !!(baseRelease.url || baseRelease.path || baseRelease.forceDeep);
}
async function scrapeRelease(baseRelease, entities, type = 'scene') {
const entity = entities.byId[baseRelease.entity?.id]
|| entities.byHostname[urlToHostname(baseRelease.url)]
|| baseRelease.entity;
if (!entity) {
logger.warn(`No entity available for ${baseRelease.url}`);
return baseRelease;
}
if ((!baseRelease.url && !baseRelease.path && !baseRelease.forceDeep) || !argv.deep) {
if (!shouldDeepScrape(baseRelease)) {
return {
...baseRelease,
entity,
@@ -247,22 +253,37 @@ async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
}
async function scrapeReleases(baseReleases, entitiesByHostname, type) {
const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesByHostname).map(async ([slug, entity]) => {
if (entity.scraper?.beforeFetchScenes) {
// entities attached to base releases (e.g. from updates) are not included in entitiesByHostname
const entitiesById = Object.fromEntries([
...Object.values(entitiesByHostname),
...baseReleases.map((baseRelease) => baseRelease.entity),
].filter(Boolean).map((entity) => [entity.id, entity]));
// only run before hook for entities that will actually be deep scraped
const deepEntityIds = new Set(baseReleases
.filter((baseRelease) => shouldDeepScrape(baseRelease))
.map((baseRelease) => baseRelease.entity?.id || entitiesByHostname[urlToHostname(baseRelease.url)]?.id)
.filter(Boolean));
const entitiesWithBeforeDataById = Object.fromEntries(await Promise.all(Object.values(entitiesById).map(async (entity) => {
if (deepEntityIds.has(entity.id) && entity.scraper?.beforeFetchScenes) {
const parameters = getRecursiveParameters(entity);
const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
return [slug, { ...entity, preData }];
return [entity.id, { ...entity, preData }];
}
return [slug, entity];
}));
return [entity.id, entity];
})));
const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean));
const entitiesWithBeforeData = {
byId: entitiesWithBeforeDataById,
byHostname: Object.fromEntries(Object.entries(entitiesByHostname).map(([hostname, entity]) => [hostname, entitiesWithBeforeDataById[entity.id]])),
};
return Promise.map(
baseReleases,
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeData, type),
{ concurrency: 1 },
);
}