Added 'newly added' filter. Handling paywalled videos in Private scraper. Added shoot ID to search.

src/media.js

@@ -271,7 +271,7 @@ function groupItems(items) {
 }
 
 async function storeMedia(sources, domain, role, { entropyFilter = 2.5 } = {}) {
-    const presentSources = sources.filter(Boolean);
+    const presentSources = sources.filter(source => typeof source === 'string' || Array.isArray(source) || (source && source.src));
 
     if (presentSources.length === 0) {
         return {};

src/releases.js

@@ -202,7 +202,7 @@ async function attachStudio(release) {
     };
 }
 
-async function curateReleaseEntry(release) {
+async function curateReleaseEntry(release, batchId, existingRelease) {
     const slug = slugify(release.title, {
         encode: true,
         limit: config.titleSlugLength,
@@ -227,6 +227,8 @@ async function curateReleaseEntry(release) {
         // rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
         deep: typeof release.deep === 'boolean' ? release.deep : false,
         deep_url: release.deepUrl,
+        updated_batch_id: batchId,
+        ...(!existingRelease && { created_batch_id: batchId }),
     };
 
     return curatedRelease;
@@ -380,6 +382,7 @@ async function updateReleasesSearch(releaseIds) {
                 sites.slug || ' ' ||
                 networks.name || ' ' ||
                 networks.slug || ' ' ||
+                releases.shoot_id || ' ' ||
                 EXTRACT(YEAR FROM releases.date) || ' ' ||
                 CAST(EXTRACT(MONTH FROM releases.date) AS VARCHAR) || ' ' ||
                 CAST(EXTRACT(DAY FROM releases.date) AS VARCHAR) || ' ' ||
@@ -408,7 +411,7 @@ async function updateReleasesSearch(releaseIds) {
     }
 }
 
-async function storeRelease(release) {
+async function storeRelease(release, batchId) {
     const existingRelease = await knex('releases')
         .where({
             entry_id: release.entryId,
@@ -416,7 +419,7 @@ async function storeRelease(release) {
         })
         .first();
 
-    const curatedRelease = await curateReleaseEntry(release);
+    const curatedRelease = await curateReleaseEntry(release, batchId, existingRelease);
 
     if (existingRelease && !argv.redownload) {
         return existingRelease;
@@ -453,11 +456,13 @@ async function storeRelease(release) {
 }
 
 async function storeReleases(releases) {
+    const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+
     const storedReleases = await Promise.map(releases, async (release) => {
         try {
             const releaseWithChannelSite = await attachChannelSite(release);
             const releaseWithStudio = await attachStudio(releaseWithChannelSite);
-            const { id, slug } = await storeRelease(releaseWithStudio);
+            const { id, slug } = await storeRelease(releaseWithStudio, batchId);
 
             return {
                 id,

src/scrapers/private.js

@@ -84,12 +84,12 @@ async function scrapeScene(html, url, site) {
     release.likes = Number($('.content-desc #social-actions #likes').text());
 
     const posterScript = $('script:contains(poster)').html();
-    const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
-    release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
+    const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
+    release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src');
 
-    release.trailer = {
-        src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
-    };
+    const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src');
+
+    if (trailer) release.trailer = { src: trailer };
 
     release.photos = await getPhotos(release.entryId, site);
     release.movie = $('a[data-track="FULL MOVIE"]').attr('href');

src/web/plugins/releases.js

@@ -4,8 +4,14 @@ const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
 
 const schemaExtender = makeExtendSchemaPlugin(_build => ({
     typeDefs: gql`
+      extend type Release {}
     `,
     resolvers: {
+        Release: {
+            async foo(_parent, _args, _context, _info) {
+                // template
+            },
+        },
     },
 }));