forked from DebaucheryLibrarian/traxxx
Added mimetype check to teasers and trailers. Added chapters to MindGeek scraper, fixed scene ID extraction getting stuck on numbers in domain name. Ordering chapters by timestamp.
This commit is contained in:
parent
a45c5f8f37
commit
c2a008afbe
|
@ -268,7 +268,7 @@ const releaseFragment = `
|
||||||
${releaseTrailerFragment}
|
${releaseTrailerFragment}
|
||||||
${releaseTeaserFragment}
|
${releaseTeaserFragment}
|
||||||
${siteFragment}
|
${siteFragment}
|
||||||
chapters {
|
chapters(orderBy: TIME_ASC) {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
time
|
time
|
||||||
|
|
|
@ -116,9 +116,9 @@ async function scrapeRelease(baseRelease, entitiesBySlug, type = 'scene') {
|
||||||
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
|
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options, null)
|
||||||
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
|
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, options, null);
|
||||||
|
|
||||||
if (typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
|
if (!scrapedRelease || typeof scrapedRelease !== 'object' || Array.isArray(scrapedRelease)) {
|
||||||
// scraper is unable to fetch the releases and returned a HTTP code or null
|
// scraper is unable to fetch the releases and returned a HTTP code or null
|
||||||
throw new Error(`Scraper returned ${scrapedRelease} when fetching latest from '${entity.name}' (${entity.parent?.name})`);
|
throw new Error(`Scraper returned '${scrapedRelease}' when fetching latest from '${entity.name}' (${entity.parent?.name})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// object-merge-advance will use null as explicit false on hard merged keys, even when null as explicit falls is disabled
|
// object-merge-advance will use null as explicit false on hard merged keys, even when null as explicit falls is disabled
|
||||||
|
|
|
@ -487,6 +487,10 @@ async function storeFile(media, options) {
|
||||||
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
|
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (['teasers', 'trailers'].includes(media.role) && media.meta.type !== 'video') {
|
||||||
|
throw new Error(`Media for '${media.role}' must be a video, but '${media.meta.mimetype}' was detected in ${media.src}`);
|
||||||
|
}
|
||||||
|
|
||||||
const [stat] = await Promise.all([
|
const [stat] = await Promise.all([
|
||||||
fsPromises.stat(media.file.path),
|
fsPromises.stat(media.file.path),
|
||||||
fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }),
|
fsPromises.mkdir(path.join(config.media.path, filedir), { recursive: true }),
|
||||||
|
|
|
@ -45,7 +45,7 @@ const releaseFields = `
|
||||||
slug
|
slug
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chapters @include (if: $full) {
|
chapters(orderBy: TIME_ASC) @include(if: $full) {
|
||||||
id
|
id
|
||||||
index
|
index
|
||||||
time
|
time
|
||||||
|
|
|
@ -84,6 +84,12 @@ function scrapeLatestX(data, site, filterChannel) {
|
||||||
if (teaser) release.teaser = teaser;
|
if (teaser) release.teaser = teaser;
|
||||||
if (trailer) release.trailer = trailer;
|
if (trailer) release.trailer = trailer;
|
||||||
|
|
||||||
|
release.chapters = data.timeTags?.map(chapter => ({
|
||||||
|
time: chapter.startTime,
|
||||||
|
duration: chapter.endTime - chapter.startTime,
|
||||||
|
tags: [chapter.name],
|
||||||
|
}));
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -114,6 +120,12 @@ function scrapeScene(data, url, _site, networkName) {
|
||||||
if (teaser) release.teaser = teaser;
|
if (teaser) release.teaser = teaser;
|
||||||
if (trailer) release.trailer = trailer;
|
if (trailer) release.trailer = trailer;
|
||||||
|
|
||||||
|
release.chapters = data.timeTags?.map(chapter => ({
|
||||||
|
time: chapter.startTime,
|
||||||
|
duration: chapter.endTime - chapter.startTime,
|
||||||
|
tags: [chapter.name],
|
||||||
|
}));
|
||||||
|
|
||||||
const siteName = data.collections[0]?.name || data.brand;
|
const siteName = data.collections[0]?.name || data.brand;
|
||||||
release.channel = slugify(siteName, '');
|
release.channel = slugify(siteName, '');
|
||||||
|
|
||||||
|
@ -279,7 +291,7 @@ async function fetchScene(url, site, baseScene, options) {
|
||||||
return baseScene;
|
return baseScene;
|
||||||
}
|
}
|
||||||
|
|
||||||
const entryId = url.match(/\d+/)[0];
|
const entryId = new URL(url).pathname.match(/\/(\d+)/)?.[1];
|
||||||
const { session, instanceToken } = await getSession(site, options.parameters);
|
const { session, instanceToken } = await getSession(site, options.parameters);
|
||||||
|
|
||||||
const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
|
const res = await http.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
|
||||||
|
|
|
@ -250,17 +250,21 @@ async function updateReleasesSearch(releaseIds) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function storeChapters(releases) {
|
async function storeChapters(releases) {
|
||||||
const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
|
const chapters = releases
|
||||||
releaseId: release.id,
|
.map(release => release.chapters?.map((chapter, index) => ({
|
||||||
index: index + 1,
|
releaseId: release.id,
|
||||||
time: chapter.time,
|
index: index + 1,
|
||||||
duration: chapter.duration,
|
time: chapter.time,
|
||||||
title: chapter.title,
|
duration: chapter.duration,
|
||||||
description: chapter.description,
|
title: chapter.title,
|
||||||
poster: chapter.poster,
|
description: chapter.description,
|
||||||
photos: chapter.photos,
|
poster: chapter.poster,
|
||||||
tags: chapter.tags,
|
photos: chapter.photos,
|
||||||
}))).flat().filter(Boolean);
|
tags: chapter.tags,
|
||||||
|
})))
|
||||||
|
.flat()
|
||||||
|
.filter(Boolean)
|
||||||
|
.sort((chapterA, chapterB) => chapterA.time - chapterB.time);
|
||||||
|
|
||||||
const curatedChapterEntries = chapters.map(chapter => ({
|
const curatedChapterEntries = chapters.map(chapter => ({
|
||||||
index: chapter.index,
|
index: chapter.index,
|
||||||
|
|
Loading…
Reference in New Issue