Added studio support. Added studios for LegalPorno. Improved media fetch error handling. Fixed DDFNetwork scraper, added media support.

src/scrapers/ddfnetwork.js

@@ -4,7 +4,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const knex = require('../knex');
+// const knex = require('../knex');
 const { matchTags } = require('../tags');
 
 /* eslint-disable newline-per-chained-call */
@@ -18,10 +18,14 @@ function scrapeLatest(html, site) {
         const url = `${site.url}${sceneLinkElement.attr('href')}`;
         const entryId = url.split('/').slice(-1)[0];
 
-        const date = moment.utc($(element).find('.card-footer .text-muted').text(), 'MMMM DD, YYYY').toDate();
-        const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray().filter(actor => actor);
+        const date = moment.utc($(element).find('small[datetime]').attr('datetime'), 'YYYY-MM-DD HH:mm:ss').toDate();
+        const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim())
+            .toArray()
+            .filter(actor => actor);
 
-        const duration = Number($(element).find('.card-info div:nth-child(2) .card-text').text().slice(0, -4)) * 60;
+        const duration = parseInt($(element).find('.card-info div:nth-child(2) .card-text').text(), 10) * 60;
+
+        const poster = sceneLinkElement.find('img').attr('data-src');
 
         return {
             url,
@@ -30,6 +34,7 @@ function scrapeLatest(html, site) {
             actors,
             date,
             duration,
+            poster,
             rating: null,
             site,
         };
@@ -40,26 +45,39 @@ async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
     const entryId = url.split('/').slice(-1)[0];
-    const title = $('.video-title h1').text();
-    const description = $('.description-box .box-container').text();
+    const title = $('meta[itemprop="name"]').attr('content');
+    const description = $('.descr-box p').text(); // meta tags don't contain full description
 
-    const date = moment.utc($('.video-title .remain time').text(), 'MMMM DD, YYYY').toDate();
-    const actors = $('.pornstars-box .pornstar-card .card-title a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
+    const date = moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate();
+    const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
 
     const likes = Number($('.info-panel.likes .likes').text());
     const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
 
+    const rawTags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
+
+    const poster = $('#video').attr('poster');
+    const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
+
+    const trailer540 = $('source[res="540"]').attr('src');
+    const trailer720 = $('source[res="720"]').attr('src');
+
+    /*
+     * broken as of nov 2019
     const { origin } = new URL($('.pornstar-card meta[itemprop="url"]').first().attr('content'));
-    const rawTags = $('#tagsBox .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
 
     const [channelSite, tags] = await Promise.all([
         // don't find site if original is already specific
         site.isFallback ? knex('sites').where({ url: origin }).first() : site,
         matchTags(rawTags),
     ]);
+    */
+
+    const tags = await matchTags(rawTags);
 
     return {
-        url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
+        // url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
+        url,
         entryId,
         title,
         description,
@@ -67,10 +85,23 @@ async function scrapeScene(html, url, site) {
         date,
         duration,
         tags,
+        poster,
+        photos,
+        trailer: trailer540
+            ? {
+                src: trailer540,
+                quality: 540,
+            }
+            : {
+                // backup
+                src: trailer720,
+                quality: 720,
+            },
         rating: {
             likes,
         },
-        site: channelSite || site,
+        // site: channelSite || site,
+        site,
     };
 }
 

src/scrapers/legalporno.js

@@ -3,6 +3,7 @@
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
+const knex = require('../knex');
 
 const { matchTags } = require('../tags');
 
@@ -95,8 +96,17 @@ async function scrapeScene(html, url, site, useGallery) {
 
     const trailer = data.clip.qualities.find(clip => clip.quality === 'vga' || clip.quality === 'hd');
 
+    const studioName = $('.watchpage-studioname').first().text().trim();
+    const studioSlug = studioName.replace(/\s+/g, '').toLowerCase();
     const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = await matchTags(rawTags);
+
+    const [studio, tags] = await Promise.all([
+        knex('studios')
+            .where({ name: studioName })
+            .orWhere({ slug: studioSlug })
+            .first(),
+        matchTags(rawTags),
+    ]);
 
     return {
         url,
@@ -115,6 +125,7 @@ async function scrapeScene(html, url, site, useGallery) {
         },
         tags,
         site,
+        studio,
     };
 }