From 7723b2b698222fd3d676bd90d9e94ae334271d9c Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 16 Jun 2023 02:29:01 +0200 Subject: [PATCH] Only curating release ID when present. --- src/scrapers/dorcel.js | 6 +++--- src/store-releases.js | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/scrapers/dorcel.js b/src/scrapers/dorcel.js index 9994a653..5e849362 100755 --- a/src/scrapers/dorcel.js +++ b/src/scrapers/dorcel.js @@ -48,7 +48,7 @@ function scrapeScene({ query }, url, channel) { release.director = query.cnt('.director')?.split(/\s*:\s*/)[1]; const fallbackPoster = query.img('.player img'); - release.poster = query.sourceSet('.player img', 'data-srcset') || [fallbackPoster.replace('_crop', ''), fallbackPoster]; + release.poster = query.sourceSet('.player img', 'data-srcset') || [fallbackPoster?.replace('_crop', ''), fallbackPoster]; const movieUrl = query.url('.movie a', 'href', { origin: channel.url }); @@ -173,7 +173,7 @@ async function fetchMovies(channel, page = 1) { Referer: 'https://www.dorcelclub.com/en/porn-movie?sorting=new', // might be used to derive sorting }); - if (res.ok) { + if (res.ok && res.items) { return scrapeMovies(res.items, channel); } @@ -199,7 +199,7 @@ async function fetchMovie(url, channel) { Referer: `${channel.url}/en/porn-movie`, }); - if (res.ok) { + if (res.ok && res.item) { return scrapeMovie(res.item, url, channel); } diff --git a/src/store-releases.js b/src/store-releases.js index 60e038d4..a3866607 100755 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -30,7 +30,6 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce }); const curatedRelease = { - id: release.id, // release is updated title: decode(release.title), entry_id: release.entryId || null, entity_id: release.entity.id, @@ -46,6 +45,11 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce updated_batch_id: batchId, }; + if (release.id) { + // release is updated + curatedRelease.id = release.id; + } + if (type === 'scene') { curatedRelease.shoot_id = release.shootId || null; curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null; @@ -474,8 +478,6 @@ async function storeScenes(releases, useBatchId) { const curatedDuplicateReleases = await Promise.all(duplicateReleasesWithId.map((release) => curateReleaseEntry(release, batchId))); const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId); - console.log(curatedDuplicateReleases); - const updated = await knex.raw(` UPDATE releases SET url = COALESCE(new.url, releases.url),