Improved media handling, added trailer support. Fetching media from Vixen network frontpages.

assets/components/home/home.vue

@@ -33,8 +33,15 @@
                         class="scene-link"
                     >
                         <img
-                            v-if="release.photos.length > 0"
+                            v-if="release.poster"
-                            :src="`/${release.photos[0].file}`"
+                            :src="`/${release.poster.path}`"
+                            :alt="release.title"
+                            class="scene-thumbnail"
+                        >
+
+                        <img
+                            v-else-if="release.photos.length > 0"
+                            :src="`/${release.photos[0].path}`"
                             :alt="release.title"
                             class="scene-thumbnail"
                         >

assets/components/release/release.vue

@@ -4,17 +4,27 @@
             class="banner"
             @wheel.prevent="scrollBanner"
         >
+            <div class="banner-trailer">
+                <video
+                    v-if="release.trailer"
+                    :src="`/${release.trailer.path}`"
+                    :poster="`/${release.poster && release.poster.path}`"
+                    :alt="release.title"
+                    class="banner-item"
+                    controls
+                >Sorry, the tailer cannot be played in your browser</video>
+            </div>
+
             <a
                 v-for="photo in photos"
                 :key="`banner-${photo.index}`"
-                :href="`/${photo.file}`"
+                :href="`/${photo.path}`"
                 target="_blank"
                 rel="noopener noreferrer"
-
             >
                 <img
-                    :src="`/${photo.file}`"
+                    :src="`/${photo.path}`"
-                    :alt="`Photo ${photo.index}`"
+                    :alt="`Photo ${photo.index + 1}`"
                     class="banner-item"
                 >
             </a>
@@ -90,12 +100,12 @@ function scrollBanner(event) {
 }
 
 function photos() {
-    if (this.release && this.release.photos.length) {
+    if (this.release.photos.length) {
-        if (this.release.photos[0].role === 'poster') {
+        return this.release.photos;
-            return this.release.photos.slice(1);
     }
 
-        return this.release.photos;
+    if (this.release.poster && !this.release.trailer) {
+        return [this.release.poster];
     }
 
     return [];
@@ -131,12 +141,17 @@ export default {
     margin: 0 0 1rem 0;
     scrollbar-width: none;
     box-shadow: 0 0 3px $shadow;
+    font-size: 0;
 
     &::-webkit-scrollbar {
         display: none;
     }
 }
 
+.banner-trailer {
+    display: inline-block;
+}
+
 .banner-item {
     height: 20rem;
     vertical-align: middle;

migrations/20190325001339_releases.js

@@ -115,14 +115,15 @@ exports.up = knex => Promise.resolve()
     .then(() => knex.schema.createTable('media', (table) => {
         table.increments('id', 16);
 
-        table.string('file');
+        table.string('path');
         table.integer('index');
         table.string('mime');
 
         table.enum('domain', ['networks', 'sites', 'releases', 'actors', 'directors']);
         table.integer('target_id', 16);
 
-        table.enum('role', ['poster', 'logo', 'profile']);
+        table.enum('role', ['photo', 'poster', 'trailer', 'logo', 'profile']);
+        table.string('quality', 6);
     }))
     .then(() => knex.schema.createTable('actors_associated', (table) => {
         table.increments('id', 16);

package-lock.json

@@ -4675,6 +4675,11 @@
           "version": "2.4.2",
           "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz",
           "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4="
+        },
+        "mime": {
+          "version": "1.6.0",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+          "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
         }
       }
     },
@@ -6969,9 +6974,9 @@
       }
     },
     "mime": {
-      "version": "1.6.0",
+      "version": "2.4.4",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
-      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
+      "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
     },
     "mime-db": {
       "version": "1.38.0",

package.json

@@ -75,6 +75,7 @@
         "fs-extra": "^7.0.1",
         "knex": "^0.16.3",
         "knex-migrate": "^1.7.1",
+        "mime": "^2.4.4",
         "moment": "^2.24.0",
         "neo-blessed": "^0.2.0",
         "opn": "^5.4.0",

public/css/style.css

@@ -114,10 +114,14 @@
   margin: 0 0 1rem 0;
   scrollbar-width: none;
   box-shadow: 0 0 3px rgba(0, 0, 0, 0.5);
+  font-size: 0;
 }
 .banner[data-v-2bc41e74]::-webkit-scrollbar {
     display: none;
 }
+.banner-trailer[data-v-2bc41e74] {
+  display: inline-block;
+}
 .banner-item[data-v-2bc41e74] {
   height: 20rem;
   vertical-align: middle;

src/fetch-releases.js

@@ -5,6 +5,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const Promise = require('bluebird');
 const moment = require('moment');
+const mime = require('mime');
 const bhttp = require('bhttp');
 
 const argv = require('./argv');
@@ -86,42 +87,71 @@ async function findDuplicateReleases(latestReleases, _siteId) {
 }
 
 async function storePhotos(release, releaseEntry) {
-    await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
+    console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
 
-    console.log(`Storing photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+    const files = await Promise.map(release.photos, async (photoUrl, index) => {
+        const { pathname } = new URL(photoUrl);
+        const mimetype = mime.getType(pathname);
 
-    const filepaths = await Promise.map(release.photos, async (photoUrl, index) => {
         const res = await bhttp.get(photoUrl);
-        const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.jpg`);
+        const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
         await fs.writeFile(path.join(config.photoPath, filepath), res.body);
 
-        return filepath;
+        return {
+            filepath,
+            mimetype,
+        };
     }, {
         concurrency: 2,
     });
 
-    await knex('media').insert(filepaths.map((filepath, index) => ({
+    await knex('media').insert(files.map(({ filepath, mimetype }, index) => ({
-        file: filepath,
+        path: filepath,
-        mime: 'image/jpeg',
+        mime: mimetype,
         index,
         domain: 'releases',
         target_id: releaseEntry.id,
-        role: null,
+        role: 'photo',
     })));
+}
 
-    if (release.trailer && release.trailer.poster) {
+async function storePoster(release, releaseEntry) {
-        const res = await bhttp.get(release.trailer.poster);
+    console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
-        const filepath = path.join(release.site.slug, releaseEntry.id.toString(), 'poster.jpg');
+
+    const { pathname } = new URL(release.poster);
+    const mimetype = mime.getType(pathname);
+
+    const res = await bhttp.get(release.poster);
+    const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `poster.${mime.getExtension(mimetype)}`);
     await fs.writeFile(path.join(config.photoPath, filepath), res.body);
 
     await knex('media').insert({
-            file: filepath,
+        path: filepath,
-            mime: 'image/jpeg',
+        mime: mimetype,
         domain: 'releases',
         target_id: releaseEntry.id,
         role: 'poster',
     });
 }
+
+async function storeTrailer(release, releaseEntry) {
+    console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+
+    const { pathname } = new URL(release.trailer.src);
+    const mimetype = release.trailer.type || mime.getType(pathname);
+
+    const res = await bhttp.get(release.trailer.src);
+    const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
+    await fs.writeFile(path.join(config.photoPath, filepath), res.body);
+
+    await knex('media').insert({
+        path: filepath,
+        mime: mimetype,
+        domain: 'releases',
+        target_id: releaseEntry.id,
+        role: 'trailer',
+        quality: release.trailer.quality || null,
+    });
 }
 
 async function storeReleases(releases = []) {
@@ -170,9 +200,21 @@ async function storeReleases(releases = []) {
             })));
         }
 
+        if (release.poster || (release.photos && release.photos.length)) {
+            await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
+        }
+
         if (release.photos && release.photos.length > 0) {
             await storePhotos(release, releaseEntry.rows[0]);
         }
+
+        if (release.poster) {
+            await storePoster(release, releaseEntry.rows[0]);
+        }
+
+        if (release.trailer) {
+            await storeTrailer(release, releaseEntry.rows[0]);
+        }
     }, {
         concurrency: 2,
     });

src/releases.js

@@ -28,7 +28,9 @@ async function curateRelease(release) {
         actors,
         director: release.director,
         tags,
-        photos: media,
+        photos: media.filter(item => item.role === 'photo'),
+        poster: media.filter(item => item.role === 'poster')[0],
+        trailer: media.filter(item => item.role === 'trailer')[0],
         rating: {
             likes: release.likes,
             dislikes: release.dislikes,

src/scrapers/kink.js

@@ -18,6 +18,7 @@ function scrapeLatest(html, site) {
         const shootId = href.split('/')[2];
         const title = sceneLinkElement.text().trim();
 
+        const poster = $(element).find('.adimage').attr('src');
         const photos = $(element).find('.rollover .roll-image').map((photoIndex, photoElement) => $(photoElement).attr('data-imagesrc')).toArray();
 
         const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
@@ -36,6 +37,7 @@ function scrapeLatest(html, site) {
             actors,
             date,
             photos,
+            poster,
             rating: {
                 stars,
             },
@@ -86,13 +88,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
         actors,
         description,
         photos,
-        trailer: {
-            video: {
-                default: trailerVideo,
-                sd: trailerVideo,
-                hd: trailerVideo.replace('480p', '720p'),
-            },
         poster: trailerPoster,
+        trailer: {
+            src: trailerVideo,
+            quality: 480,
         },
         rating: {
             stars,

src/scrapers/vixen.js

@@ -15,18 +15,32 @@ function scrapeLatest(html, site) {
 
     return scenes.map((scene) => {
         const shootId = String(scene.newId);
-        const { title } = scene;
+
+        const {
+            title,
+            models: actors,
+        } = scene;
+
         const url = `${site.url}${scene.targetUrl}`;
         const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
-        const actors = scene.models;
         const stars = Number(scene.textRating) / 2;
 
+        // largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
+        const poster = scene.images.listing.slice(-1)[0].src;
+        const trailer = scene.previews.listing.slice(-1)[0];
+
         return {
             url,
             shootId,
             title,
             actors,
             date,
+            poster,
+            trailer: {
+                src: trailer.src,
+                type: trailer.type,
+                quality: trailer.height,
+            },
             rating: {
                 stars,
             },
@@ -45,21 +59,21 @@ async function scrapeScene(html, url, site) {
     const shootId = data.page.data[`${pathname}${search}`].data.video;
     const scene = data.videos.find(video => video.newId === shootId);
 
+    // console.log(scene);
+
     const {
         title,
         description,
         models: actors,
         totalRateVal: stars,
+        runLength: duration,
+        directorNames: director,
+        tags: rawTags,
     } = scene;
 
     const date = new Date(scene.releaseDate);
-
-    const rawTags = scene.tags;
     const tags = await matchTags(rawTags);
 
-    const duration = scene.runLength;
-    const director = scene.directorNames;
-
     return {
         url,
         shootId,