Added mimetype check to teasers and trailers. Added chapters to MindGeek scraper, fixed scene ID extraction getting stuck on numbers in domain name. Ordering chapters by timestamp.

This commit is contained in:
DebaucheryLibrarian 2021-02-27 18:05:06 +01:00
parent a45c5f8f37
commit c2a008afbe
6 changed files with 36 additions and 16 deletions

View File

@ -268,7 +268,7 @@ const releaseFragment = `
${releaseTrailerFragment} ${releaseTrailerFragment}
${releaseTeaserFragment} ${releaseTeaserFragment}
${siteFragment} ${siteFragment}
chapters { chapters(orderBy: TIME_ASC) {
id id
index index
time time

View File

@ -116,9 +116,9 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null) ? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null); : await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
if (typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) { if (!scrapedRelease || typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
// scraper is unable to fetch the releases and returned a HTTP code or null // scraper is unable to fetch the releases and returned a HTTP code or null
throw new Error(`Scraper returned ${scrapedRelease} when fetching latest from '${entity.name}' (${entity.parent?.name})`); throw new Error(`Scraper returned '${scrapedRelease}' when fetching latest from '${entity.name}' (${entity.parent?.name})`);
} }
// object-merge-advance will use null as explicit false on hard merged keys, even when null as explicit falls is disabled // object-merge-advance will use null as explicit false on hard merged keys, even when null as explicit falls is disabled

View File

@ -487,6 +487,10 @@ async function storeFile(media, options) {
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`); throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
} }
if (['teasers', 'trailers'].includes(media.role) && media.meta.type !== 'video') {
throw new Error(`Media for '${media.role}' must be a video, but '${media.meta.mimetype}' was detected in ${media.src}`);
}
const [stat] = await Promise.all([ const [stat] = await Promise.all([
fsPromises.stat(media.file.path), fsPromises.stat(media.file.path),
fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }), fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }),

View File

@ -45,7 +45,7 @@ const releaseFields = `
slug slug
} }
} }
chapters @include (if: $full) { chapters(orderBy: TIME_ASC) @include(if: $full) {
id id
index index
time time

View File

@ -84,6 +84,12 @@ function scrapeLatestX(data, site, filterChannel) {
if (teaser) release.teaser = teaser; if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer; if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
return release; return release;
} }
@ -114,6 +120,12 @@ function scrapeScene(data, url, _site, networkName) {
if (teaser) release.teaser = teaser; if (teaser) release.teaser = teaser;
if (trailer) release.trailer = trailer; if (trailer) release.trailer = trailer;
release.chapters = data.timeTags?.map(chapter => ({
time: chapter.startTime,
duration: chapter.endTime - chapter.startTime,
tags: [chapter.name],
}));
const siteName = data.collections[0]?.name || data.brand; const siteName = data.collections[0]?.name || data.brand;
release.channel = slugify(siteName, ''); release.channel = slugify(siteName, '');
@ -279,7 +291,7 @@ async function fetchScene(url, site, baseScene, options) {
return baseScene; return baseScene;
} }
const entryId = url.match(/\d+/)[0]; const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
const { session, instanceToken } = await getSession(site, options.parameters); const { session, instanceToken } = await getSession(site, options.parameters);
const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, { const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {

View File

@ -250,17 +250,21 @@ async function updateReleasesSearch(releaseIds) {
} }
async function storeChapters(releases) { async function storeChapters(releases) {
const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({ const chapters = releases
releaseId: release.id, .map(release => release.chapters?.map((chapter, index) => ({
index: index + 1, releaseId: release.id,
time: chapter.time, index: index + 1,
duration: chapter.duration, time: chapter.time,
title: chapter.title, duration: chapter.duration,
description: chapter.description, title: chapter.title,
poster: chapter.poster, description: chapter.description,
photos: chapter.photos, poster: chapter.poster,
tags: chapter.tags, photos: chapter.photos,
}))).flat().filter(Boolean); tags: chapter.tags,
})))
.flat()
.filter(Boolean)
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
const curatedChapterEntries = chapters.map(chapter => ({ const curatedChapterEntries = chapters.map(chapter => ({
index: chapter.index, index: chapter.index,