DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Improved media handling, added trailer support. Fetching media from Vixen network frontpages.

This commit is contained in:
ThePendulum 2019-09-25 04:52:58 +02:00
parent b12d74cbb3
commit 96d8cfbcb6
10 changed files with 141 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
assets/components/home/home.vue
View File

@ -33,8 +33,15 @@
class="scene-link"
>
<img
v-if="release.photos.length > 0"
:src="`/${release.photos[0].file}`"
v-if="release.poster"
:src="`/${release.poster.path}`"
:alt="release.title"
class="scene-thumbnail"
>
<img
v-else-if="release.photos.length > 0"
:src="`/${release.photos[0].path}`"
:alt="release.title"
class="scene-thumbnail"
>

33
assets/components/release/release.vue
View File

@ -4,17 +4,27 @@
class="banner"
@wheel.prevent="scrollBanner"
>
<div class="banner-trailer">
<video
v-if="release.trailer"
:src="`/${release.trailer.path}`"
:poster="`/${release.poster && release.poster.path}`"
:alt="release.title"
class="banner-item"
controls
>Sorry, the tailer cannot be played in your browser</video>
</div>
<a
v-for="photo in photos"
:key="`banner-${photo.index}`"
:href="`/${photo.file}`"
:href="`/${photo.path}`"
target="_blank"
rel="noopener noreferrer"
>
<img
:src="`/${photo.file}`"
:alt="`Photo ${photo.index}`"
:src="`/${photo.path}`"
:alt="`Photo ${photo.index + 1}`"
class="banner-item"
>
</a>
@ -90,14 +100,14 @@ function scrollBanner(event) {
}
function photos() {
if (this.release && this.release.photos.length) {
if (this.release.photos[0].role === 'poster') {
return this.release.photos.slice(1);
}
if (this.release.photos.length) {
return this.release.photos;
}
if (this.release.poster && !this.release.trailer) {
return [this.release.poster];
}
return [];
}
@ -131,12 +141,17 @@ export default {
margin: 0 0 1rem 0;
scrollbar-width: none;
box-shadow: 0 0 3px $shadow;
font-size: 0;
&::-webkit-scrollbar {
display: none;
}
}
.banner-trailer {
display: inline-block;
}
.banner-item {
height: 20rem;
vertical-align: middle;

5
migrations/20190325001339_releases.js
View File

@ -115,14 +115,15 @@ exports.up = knex => Promise.resolve()
.then(() => knex.schema.createTable('media', (table) => {
table.increments('id', 16);
table.string('file');
table.string('path');
table.integer('index');
table.string('mime');
table.enum('domain', ['networks', 'sites', 'releases', 'actors', 'directors']);
table.integer('target_id', 16);
table.enum('role', ['poster', 'logo', 'profile']);
table.enum('role', ['photo', 'poster', 'trailer', 'logo', 'profile']);
table.string('quality', 6);
}))
.then(() => knex.schema.createTable('actors_associated', (table) => {
table.increments('id', 16);

11
package-lock.json generated
View File

@ -4675,6 +4675,11 @@
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz",
"integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4="
},
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
}
}
},
@ -6969,9 +6974,9 @@
}
},
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
"version": "2.4.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
"integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
},
"mime-db": {
"version": "1.38.0",

1
package.json
View File

@ -75,6 +75,7 @@
"fs-extra": "^7.0.1",
"knex": "^0.16.3",
"knex-migrate": "^1.7.1",
"mime": "^2.4.4",
"moment": "^2.24.0",
"neo-blessed": "^0.2.0",
"opn": "^5.4.0",

4
public/css/style.css
View File

@ -114,10 +114,14 @@
margin: 0 0 1rem 0;
scrollbar-width: none;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.5);
font-size: 0;
}
.banner[data-v-2bc41e74]::-webkit-scrollbar {
display: none;
}
.banner-trailer[data-v-2bc41e74] {
display: inline-block;
}
.banner-item[data-v-2bc41e74] {
height: 20rem;
vertical-align: middle;

84
src/fetch-releases.js
View File

@ -5,6 +5,7 @@ const fs = require('fs-extra');
const path = require('path');
const Promise = require('bluebird');
const moment = require('moment');
const mime = require('mime');
const bhttp = require('bhttp');
const argv = require('./argv');
@ -86,42 +87,71 @@ async function findDuplicateReleases(latestReleases, _siteId) {
}
async function storePhotos(release, releaseEntry) {
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
const files = await Promise.map(release.photos, async (photoUrl, index) => {
const { pathname } = new URL(photoUrl);
const mimetype = mime.getType(pathname);
const filepaths = await Promise.map(release.photos, async (photoUrl, index) => {
const res = await bhttp.get(photoUrl);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.jpg`);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
return filepath;
return {
filepath,
mimetype,
};
}, {
concurrency: 2,
});
await knex('media').insert(filepaths.map((filepath, index) => ({
file: filepath,
mime: 'image/jpeg',
await knex('media').insert(files.map(({ filepath, mimetype }, index) => ({
path: filepath,
mime: mimetype,
index,
domain: 'releases',
target_id: releaseEntry.id,
role: null,
role: 'photo',
})));
}
if (release.trailer && release.trailer.poster) {
const res = await bhttp.get(release.trailer.poster);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), 'poster.jpg');
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
async function storePoster(release, releaseEntry) {
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
await knex('media').insert({
file: filepath,
mime: 'image/jpeg',
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
});
}
const { pathname } = new URL(release.poster);
const mimetype = mime.getType(pathname);
const res = await bhttp.get(release.poster);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `poster.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
await knex('media').insert({
path: filepath,
mime: mimetype,
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
});
}
async function storeTrailer(release, releaseEntry) {
console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
const { pathname } = new URL(release.trailer.src);
const mimetype = release.trailer.type || mime.getType(pathname);
const res = await bhttp.get(release.trailer.src);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
await knex('media').insert({
path: filepath,
mime: mimetype,
domain: 'releases',
target_id: releaseEntry.id,
role: 'trailer',
quality: release.trailer.quality || null,
});
}
async function storeReleases(releases = []) {
@ -170,9 +200,21 @@ async function storeReleases(releases = []) {
})));
}
if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
}
if (release.photos && release.photos.length > 0) {
await storePhotos(release, releaseEntry.rows[0]);
}
if (release.poster) {
await storePoster(release, releaseEntry.rows[0]);
}
if (release.trailer) {
await storeTrailer(release, releaseEntry.rows[0]);
}
}, {
concurrency: 2,
});

4
src/releases.js
View File

@ -28,7 +28,9 @@ async function curateRelease(release) {
actors,
director: release.director,
tags,
photos: media,
photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0],
trailer: media.filter(item => item.role === 'trailer')[0],
rating: {
likes: release.likes,
dislikes: release.dislikes,

11
src/scrapers/kink.js
View File

@ -18,6 +18,7 @@ function scrapeLatest(html, site) {
const shootId = href.split('/')[2];
const title = sceneLinkElement.text().trim();
const poster = $(element).find('.adimage').attr('src');
const photos = $(element).find('.rollover .roll-image').map((photoIndex, photoElement) => $(photoElement).attr('data-imagesrc')).toArray();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
@ -36,6 +37,7 @@ function scrapeLatest(html, site) {
actors,
date,
photos,
poster,
rating: {
stars,
},
@ -86,13 +88,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
actors,
description,
photos,
poster: trailerPoster,
trailer: {
video: {
default: trailerVideo,
sd: trailerVideo,
hd: trailerVideo.replace('480p', '720p'),
},
poster: trailerPoster,
src: trailerVideo,
quality: 480,
},
rating: {
stars,

28
src/scrapers/vixen.js
View File

@ -15,18 +15,32 @@ function scrapeLatest(html, site) {
return scenes.map((scene) => {
const shootId = String(scene.newId);
const { title } = scene;
const {
title,
models: actors,
} = scene;
const url = `${site.url}${scene.targetUrl}`;
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
const actors = scene.models;
const stars = Number(scene.textRating) / 2;
// largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
const poster = scene.images.listing.slice(-1)[0].src;
const trailer = scene.previews.listing.slice(-1)[0];
return {
url,
shootId,
title,
actors,
date,
poster,
trailer: {
src: trailer.src,
type: trailer.type,
quality: trailer.height,
},
rating: {
stars,
},
@ -45,21 +59,21 @@ async function scrapeScene(html, url, site) {
const shootId = data.page.data[`${pathname}${search}`].data.video;
const scene = data.videos.find(video => video.newId === shootId);
// console.log(scene);
const {
title,
description,
models: actors,
totalRateVal: stars,
runLength: duration,
directorNames: director,
tags: rawTags,
} = scene;
const date = new Date(scene.releaseDate);
const rawTags = scene.tags;
const tags = await matchTags(rawTags);
const duration = scene.runLength;
const director = scene.directorNames;
return {
url,
shootId,