Updated Naughty America scraper, added media support..

migrations/20190325001339_releases.js

@@ -184,7 +184,6 @@ exports.up = knex => Promise.resolve()
 
         table.string('shoot_id');
         table.string('entry_id');
-        table.unique(['site_id', 'shoot_id']);
         table.unique(['site_id', 'entry_id']);
 
         table.string('url', 1000);

src/releases.js

@@ -223,12 +223,16 @@ async function fetchTagReleases(queryObject, options = {}) {
 async function storeReleaseAssets(release, releaseId) {
     await createReleaseMediaDirectory(release, releaseId);
 
+    try {
         await Promise.all([
             associateTags(release, releaseId),
             storePhotos(release, releaseId),
             storePoster(release, releaseId),
             storeTrailer(release, releaseId),
         ]);
+    } catch (error) {
+        console.log(release, error);
+    }
 }
 
 async function storeRelease(release) {

src/scrapers/naughtyamerica.js

@@ -5,9 +5,6 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const knex = require('../knex');
-const { matchTags } = require('../tags');
-
 function titleExtractor(pathname) {
     const components = pathname.split('/')[2].split('-');
     const entryId = components.slice(-1)[0];
@@ -34,6 +31,9 @@ function scrapeLatest(html, site) {
 
         const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
 
+        const posterString = sceneLinkElement.find('img[data-srcset]').attr('data-srcset') || sceneLinkElement.find('img[data-src]').attr('data-src');
+        const poster = `https:${posterString.match(/[\w/.]+$/)[0]}`;
+
         return {
             url,
             entryId,
@@ -41,6 +41,7 @@ function scrapeLatest(html, site) {
             actors,
             date,
             duration,
+            poster,
             rating: null,
             site,
         };
@@ -63,18 +64,17 @@ async function scrapeScene(html, url, site) {
 
     const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;
 
+    const poster = `https:${$('video, dl8-video').attr('poster')}`;
+    const photos = $('.contain-scene-images.desktop-only a').map((index, el) => `https:${$(el).attr('href')}`).toArray();
+
+    const trailerEl = $('source');
+    const trailerSrc = trailerEl.attr('src');
+    const trailerType = trailerEl.attr('type');
+
     const siteName = sceneElement.find('a.site-title').text();
-    const siteId = siteName.replace(/[\s']+/g, '').toLowerCase();
+    const channel = siteName.replace(/[\s']+/g, '').toLowerCase();
 
-    const rawTags = $('.categories a.cat-tag').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
+    const tags = $('.categories a.cat-tag').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-
-    const [channelSite, tags] = await Promise.all([
-        knex('sites')
-            .where({ slug: siteId })
-            .orWhere({ name: siteName })
-            .first(),
-        matchTags(rawTags),
-    ]);
 
     return {
         url,
@@ -85,8 +85,15 @@ async function scrapeScene(html, url, site) {
         date,
         duration,
         tags,
+        photos,
+        poster,
+        trailer: {
+            src: trailerSrc,
+            type: trailerType,
+        },
         rating: null,
-        site: channelSite || site,
+        site,
+        channel,
     };
 }