Improved media handling, added trailer support. Fetching media from Vixen network frontpages.
This commit is contained in:
parent
b12d74cbb3
commit
96d8cfbcb6
|
|
@ -33,8 +33,15 @@
|
|||
class="scene-link"
|
||||
>
|
||||
<img
|
||||
v-if="release.photos.length > 0"
|
||||
:src="`/${release.photos[0].file}`"
|
||||
v-if="release.poster"
|
||||
:src="`/${release.poster.path}`"
|
||||
:alt="release.title"
|
||||
class="scene-thumbnail"
|
||||
>
|
||||
|
||||
<img
|
||||
v-else-if="release.photos.length > 0"
|
||||
:src="`/${release.photos[0].path}`"
|
||||
:alt="release.title"
|
||||
class="scene-thumbnail"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -4,17 +4,27 @@
|
|||
class="banner"
|
||||
@wheel.prevent="scrollBanner"
|
||||
>
|
||||
<div class="banner-trailer">
|
||||
<video
|
||||
v-if="release.trailer"
|
||||
:src="`/${release.trailer.path}`"
|
||||
:poster="`/${release.poster && release.poster.path}`"
|
||||
:alt="release.title"
|
||||
class="banner-item"
|
||||
controls
|
||||
>Sorry, the tailer cannot be played in your browser</video>
|
||||
</div>
|
||||
|
||||
<a
|
||||
v-for="photo in photos"
|
||||
:key="`banner-${photo.index}`"
|
||||
:href="`/${photo.file}`"
|
||||
:href="`/${photo.path}`"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
|
||||
>
|
||||
<img
|
||||
:src="`/${photo.file}`"
|
||||
:alt="`Photo ${photo.index}`"
|
||||
:src="`/${photo.path}`"
|
||||
:alt="`Photo ${photo.index + 1}`"
|
||||
class="banner-item"
|
||||
>
|
||||
</a>
|
||||
|
|
@ -90,12 +100,12 @@ function scrollBanner(event) {
|
|||
}
|
||||
|
||||
function photos() {
|
||||
if (this.release && this.release.photos.length) {
|
||||
if (this.release.photos[0].role === 'poster') {
|
||||
return this.release.photos.slice(1);
|
||||
if (this.release.photos.length) {
|
||||
return this.release.photos;
|
||||
}
|
||||
|
||||
return this.release.photos;
|
||||
if (this.release.poster && !this.release.trailer) {
|
||||
return [this.release.poster];
|
||||
}
|
||||
|
||||
return [];
|
||||
|
|
@ -131,12 +141,17 @@ export default {
|
|||
margin: 0 0 1rem 0;
|
||||
scrollbar-width: none;
|
||||
box-shadow: 0 0 3px $shadow;
|
||||
font-size: 0;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
.banner-trailer {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.banner-item {
|
||||
height: 20rem;
|
||||
vertical-align: middle;
|
||||
|
|
|
|||
|
|
@ -115,14 +115,15 @@ exports.up = knex => Promise.resolve()
|
|||
.then(() => knex.schema.createTable('media', (table) => {
|
||||
table.increments('id', 16);
|
||||
|
||||
table.string('file');
|
||||
table.string('path');
|
||||
table.integer('index');
|
||||
table.string('mime');
|
||||
|
||||
table.enum('domain', ['networks', 'sites', 'releases', 'actors', 'directors']);
|
||||
table.integer('target_id', 16);
|
||||
|
||||
table.enum('role', ['poster', 'logo', 'profile']);
|
||||
table.enum('role', ['photo', 'poster', 'trailer', 'logo', 'profile']);
|
||||
table.string('quality', 6);
|
||||
}))
|
||||
.then(() => knex.schema.createTable('actors_associated', (table) => {
|
||||
table.increments('id', 16);
|
||||
|
|
|
|||
|
|
@ -4675,6 +4675,11 @@
|
|||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz",
|
||||
"integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4="
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -6969,9 +6974,9 @@
|
|||
}
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
|
||||
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
|
||||
"version": "2.4.4",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
|
||||
"integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.38.0",
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@
|
|||
"fs-extra": "^7.0.1",
|
||||
"knex": "^0.16.3",
|
||||
"knex-migrate": "^1.7.1",
|
||||
"mime": "^2.4.4",
|
||||
"moment": "^2.24.0",
|
||||
"neo-blessed": "^0.2.0",
|
||||
"opn": "^5.4.0",
|
||||
|
|
|
|||
|
|
@ -114,10 +114,14 @@
|
|||
margin: 0 0 1rem 0;
|
||||
scrollbar-width: none;
|
||||
box-shadow: 0 0 3px rgba(0, 0, 0, 0.5);
|
||||
font-size: 0;
|
||||
}
|
||||
.banner[data-v-2bc41e74]::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
.banner-trailer[data-v-2bc41e74] {
|
||||
display: inline-block;
|
||||
}
|
||||
.banner-item[data-v-2bc41e74] {
|
||||
height: 20rem;
|
||||
vertical-align: middle;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ const fs = require('fs-extra');
|
|||
const path = require('path');
|
||||
const Promise = require('bluebird');
|
||||
const moment = require('moment');
|
||||
const mime = require('mime');
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
const argv = require('./argv');
|
||||
|
|
@ -86,42 +87,71 @@ async function findDuplicateReleases(latestReleases, _siteId) {
|
|||
}
|
||||
|
||||
async function storePhotos(release, releaseEntry) {
|
||||
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
|
||||
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
|
||||
|
||||
console.log(`Storing photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
|
||||
const files = await Promise.map(release.photos, async (photoUrl, index) => {
|
||||
const { pathname } = new URL(photoUrl);
|
||||
const mimetype = mime.getType(pathname);
|
||||
|
||||
const filepaths = await Promise.map(release.photos, async (photoUrl, index) => {
|
||||
const res = await bhttp.get(photoUrl);
|
||||
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.jpg`);
|
||||
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
|
||||
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
|
||||
|
||||
return filepath;
|
||||
return {
|
||||
filepath,
|
||||
mimetype,
|
||||
};
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
||||
await knex('media').insert(filepaths.map((filepath, index) => ({
|
||||
file: filepath,
|
||||
mime: 'image/jpeg',
|
||||
await knex('media').insert(files.map(({ filepath, mimetype }, index) => ({
|
||||
path: filepath,
|
||||
mime: mimetype,
|
||||
index,
|
||||
domain: 'releases',
|
||||
target_id: releaseEntry.id,
|
||||
role: null,
|
||||
role: 'photo',
|
||||
})));
|
||||
}
|
||||
|
||||
if (release.trailer && release.trailer.poster) {
|
||||
const res = await bhttp.get(release.trailer.poster);
|
||||
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), 'poster.jpg');
|
||||
async function storePoster(release, releaseEntry) {
|
||||
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
|
||||
|
||||
const { pathname } = new URL(release.poster);
|
||||
const mimetype = mime.getType(pathname);
|
||||
|
||||
const res = await bhttp.get(release.poster);
|
||||
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `poster.${mime.getExtension(mimetype)}`);
|
||||
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
|
||||
|
||||
await knex('media').insert({
|
||||
file: filepath,
|
||||
mime: 'image/jpeg',
|
||||
path: filepath,
|
||||
mime: mimetype,
|
||||
domain: 'releases',
|
||||
target_id: releaseEntry.id,
|
||||
role: 'poster',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function storeTrailer(release, releaseEntry) {
|
||||
console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
|
||||
|
||||
const { pathname } = new URL(release.trailer.src);
|
||||
const mimetype = release.trailer.type || mime.getType(pathname);
|
||||
|
||||
const res = await bhttp.get(release.trailer.src);
|
||||
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
||||
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
|
||||
|
||||
await knex('media').insert({
|
||||
path: filepath,
|
||||
mime: mimetype,
|
||||
domain: 'releases',
|
||||
target_id: releaseEntry.id,
|
||||
role: 'trailer',
|
||||
quality: release.trailer.quality || null,
|
||||
});
|
||||
}
|
||||
|
||||
async function storeReleases(releases = []) {
|
||||
|
|
@ -170,9 +200,21 @@ async function storeReleases(releases = []) {
|
|||
})));
|
||||
}
|
||||
|
||||
if (release.poster || (release.photos && release.photos.length)) {
|
||||
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
|
||||
}
|
||||
|
||||
if (release.photos && release.photos.length > 0) {
|
||||
await storePhotos(release, releaseEntry.rows[0]);
|
||||
}
|
||||
|
||||
if (release.poster) {
|
||||
await storePoster(release, releaseEntry.rows[0]);
|
||||
}
|
||||
|
||||
if (release.trailer) {
|
||||
await storeTrailer(release, releaseEntry.rows[0]);
|
||||
}
|
||||
}, {
|
||||
concurrency: 2,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ async function curateRelease(release) {
|
|||
actors,
|
||||
director: release.director,
|
||||
tags,
|
||||
photos: media,
|
||||
photos: media.filter(item => item.role === 'photo'),
|
||||
poster: media.filter(item => item.role === 'poster')[0],
|
||||
trailer: media.filter(item => item.role === 'trailer')[0],
|
||||
rating: {
|
||||
likes: release.likes,
|
||||
dislikes: release.dislikes,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ function scrapeLatest(html, site) {
|
|||
const shootId = href.split('/')[2];
|
||||
const title = sceneLinkElement.text().trim();
|
||||
|
||||
const poster = $(element).find('.adimage').attr('src');
|
||||
const photos = $(element).find('.rollover .roll-image').map((photoIndex, photoElement) => $(photoElement).attr('data-imagesrc')).toArray();
|
||||
|
||||
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
|
||||
|
|
@ -36,6 +37,7 @@ function scrapeLatest(html, site) {
|
|||
actors,
|
||||
date,
|
||||
photos,
|
||||
poster,
|
||||
rating: {
|
||||
stars,
|
||||
},
|
||||
|
|
@ -86,13 +88,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
|
|||
actors,
|
||||
description,
|
||||
photos,
|
||||
trailer: {
|
||||
video: {
|
||||
default: trailerVideo,
|
||||
sd: trailerVideo,
|
||||
hd: trailerVideo.replace('480p', '720p'),
|
||||
},
|
||||
poster: trailerPoster,
|
||||
trailer: {
|
||||
src: trailerVideo,
|
||||
quality: 480,
|
||||
},
|
||||
rating: {
|
||||
stars,
|
||||
|
|
|
|||
|
|
@ -15,18 +15,32 @@ function scrapeLatest(html, site) {
|
|||
|
||||
return scenes.map((scene) => {
|
||||
const shootId = String(scene.newId);
|
||||
const { title } = scene;
|
||||
|
||||
const {
|
||||
title,
|
||||
models: actors,
|
||||
} = scene;
|
||||
|
||||
const url = `${site.url}${scene.targetUrl}`;
|
||||
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
|
||||
const actors = scene.models;
|
||||
const stars = Number(scene.textRating) / 2;
|
||||
|
||||
// largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
|
||||
const poster = scene.images.listing.slice(-1)[0].src;
|
||||
const trailer = scene.previews.listing.slice(-1)[0];
|
||||
|
||||
return {
|
||||
url,
|
||||
shootId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
poster,
|
||||
trailer: {
|
||||
src: trailer.src,
|
||||
type: trailer.type,
|
||||
quality: trailer.height,
|
||||
},
|
||||
rating: {
|
||||
stars,
|
||||
},
|
||||
|
|
@ -45,21 +59,21 @@ async function scrapeScene(html, url, site) {
|
|||
const shootId = data.page.data[`${pathname}${search}`].data.video;
|
||||
const scene = data.videos.find(video => video.newId === shootId);
|
||||
|
||||
// console.log(scene);
|
||||
|
||||
const {
|
||||
title,
|
||||
description,
|
||||
models: actors,
|
||||
totalRateVal: stars,
|
||||
runLength: duration,
|
||||
directorNames: director,
|
||||
tags: rawTags,
|
||||
} = scene;
|
||||
|
||||
const date = new Date(scene.releaseDate);
|
||||
|
||||
const rawTags = scene.tags;
|
||||
const tags = await matchTags(rawTags);
|
||||
|
||||
const duration = scene.runLength;
|
||||
const director = scene.directorNames;
|
||||
|
||||
return {
|
||||
url,
|
||||
shootId,
|
||||
|
|
|
|||
Loading…
Reference in New Issue