Fixed Whale Member posters and photos.

This commit is contained in:
2020-05-20 02:23:45 +02:00
parent b6691e1991
commit b1b7cd6d50
4 changed files with 26 additions and 11 deletions

View File

@@ -14,7 +14,8 @@ function urlToSiteSlug(url) {
try {
const slug = new URL(url)
.hostname
.match(/([\w-]+)\.\w+$/)?.[1];
.match(/([\w-]+)\.\w+$/)?.[1]
.replace(/[-_]+/g, '');
return slug;
} catch (error) {
@@ -90,7 +91,6 @@ function toBaseReleases(baseReleasesOrUrls) {
async function scrapeRelease(baseRelease, sites, type = 'scene') {
const site = baseRelease.site || sites[urlToSiteSlug(baseRelease.url)];
const siteWithFallbackNetwork = site.isNetwork ? { ...site, network: site } : site; // make site.network available, even when site is network fallback
if (!site) {
logger.warn(`No site available for ${baseRelease.url}`);
@@ -104,6 +104,7 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
};
}
const siteWithFallbackNetwork = site.isNetwork ? { ...site, network: site } : site; // make site.network available, even when site is network fallback
const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
if (!scraper) {
@@ -131,7 +132,7 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
};
if (!mergedRelease.entryId) {
throw new Error('No entry ID supplied');
throw Object.assign(new Error('No entry ID supplied'), { code: 'NO_ENTRY_ID' });
}
if (scrapedRelease && baseRelease?.tags) {
@@ -142,6 +143,11 @@ async function scrapeRelease(baseRelease, sites, type = 'scene') {
return mergedRelease;
} catch (error) {
logger.error(`Deep scrape failed for ${baseRelease.url}: ${error.message}`);
if (error.code === 'NO_ENTRY_ID') {
return null;
}
return baseRelease;
}
}
@@ -160,7 +166,7 @@ async function fetchReleases(baseReleasesOrUrls, type = 'scene') {
const deepReleases = await scrapeReleases(baseReleases, sites, type);
return deepReleases;
return deepReleases.filter(Boolean);
}
async function fetchScenes(baseReleasesOrUrls) {