Added mimetype check to teasers and trailers. Added chapters to MindGeek scraper, fixed scene ID extraction getting stuck on numbers in domain name. Ordering chapters by timestamp.

This commit is contained in:
DebaucheryLibrarian 2021-02-27 18:05:06 +01:00
parent a45c5f8f37
commit c2a008afbe
6 changed files with 36 additions and 16 deletions

View File

@ -268,7 +268,7 @@ const releaseFragment = `
${releaseTrailerFragment}
${releaseTeaserFragment}
${siteFragment}
chapters {
chapters(orderBy: TIME_ASC) {
id
index
time

View File

@ -116,9 +116,9 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
if (typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
if (!scrapedRelease || typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
// scraper is unable to fetch the releases and returned a HTTP code or null
throw new Error(`Scraper returned ${scrapedRelease} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
throw new Error(`Scraper returned '${scrapedRelease}' when fetching latest from '${entity.name}' (${entity.parent?.name})`);
}
// object-merge-advance will use null as explicit false on hard merged keys, even when null as explicit falls is disabled

View File

@ -487,6 +487,10 @@ async function storeFile(media, options) {
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
}
if (['teasers', 'trailers'].includes(media.role) && media.meta.type !== 'video') {
throw new Error(`Media for '${media.role}' must be a video, but '${media.meta.mimetype}' was detected in ${media.src}`);
}
const [stat] = await Promise.all([
fsPromises.stat(media.file.path),
fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }),

View File

@ -45,7 +45,7 @@ const releaseFields = `
slug
}
}
chapters @include (if: $full) {
chapters(orderBy: TIME_ASC) @include(if: $full) {
id
index
time

View File

@ -84,6 +84,12 @@ function scrapeLatestX(data, site, filterChannel) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
return release;
}
@ -114,6 +120,12 @@ function scrapeScene(data, url, _site, networkName) {
if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
const siteName = data.collections[0]?.name || data.brand;
release.channel = slugify(siteName, '');
@ -279,7 +291,7 @@ async function fetchScene(url, site, baseScene, options) {
return baseScene;
}
const entryId = url.match(/\d+/)[0];
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
const { session, instanceToken } = await getSession(site, options.parameters);
const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {

View File

@ -250,7 +250,8 @@ async function updateReleasesSearch(releaseIds) {
}
async function storeChapters(releases) {
const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
const chapters = releases
.map(release => release.chapters?.map((chapter, index) => ({
releaseId: release.id,
index: index + 1,
time: chapter.time,
@ -260,7 +261,10 @@ async function storeChapters(releases) {
poster: chapter.poster,
photos: chapter.photos,
tags: chapter.tags,
}))).flat().filter(Boolean);
})))
.flat()
.filter(Boolean)
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
const curatedChapterEntries = chapters.map(chapter => ({
index: chapter.index,