DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Improved media handling, added trailer support. Fetching media from Vixen network frontpages.

Browse Source
This commit is contained in:
ThePendulum
2019-09-25 04:52:58 +02:00
parent b12d74cbb3
commit 96d8cfbcb6
10 changed files with 141 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/fetch-releases.js
View File

@@ -5,6 +5,7 @@ const fs = require('fs-extra');
const path = require('path');
const Promise = require('bluebird');
const moment = require('moment');
const mime = require('mime');
const bhttp = require('bhttp');
const argv = require('./argv');
@@ -86,42 +87,71 @@ async function findDuplicateReleases(latestReleases, _siteId) {
}
async function storePhotos(release, releaseEntry) {
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.id.toString()), { recursive: true });
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
const files = await Promise.map(release.photos, async (photoUrl, index) => {
const { pathname } = new URL(photoUrl);
const mimetype = mime.getType(pathname);
const filepaths = await Promise.map(release.photos, async (photoUrl, index) => {
const res = await bhttp.get(photoUrl);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.jpg`);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `${index + 1}.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
return filepath;
return {
filepath,
mimetype,
};
}, {
concurrency: 2,
});
await knex('media').insert(filepaths.map((filepath, index) => ({
file: filepath,
mime: 'image/jpeg',
await knex('media').insert(files.map(({ filepath, mimetype }, index) => ({
path: filepath,
mime: mimetype,
index,
domain: 'releases',
target_id: releaseEntry.id,
role: null,
role: 'photo',
})));
}
if (release.trailer && release.trailer.poster) {
const res = await bhttp.get(release.trailer.poster);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), 'poster.jpg');
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
async function storePoster(release, releaseEntry) {
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
await knex('media').insert({
file: filepath,
mime: 'image/jpeg',
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
});
}
const { pathname } = new URL(release.poster);
const mimetype = mime.getType(pathname);
const res = await bhttp.get(release.poster);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `poster.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
await knex('media').insert({
path: filepath,
mime: mimetype,
domain: 'releases',
target_id: releaseEntry.id,
role: 'poster',
});
}
async function storeTrailer(release, releaseEntry) {
console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
const { pathname } = new URL(release.trailer.src);
const mimetype = release.trailer.type || mime.getType(pathname);
const res = await bhttp.get(release.trailer.src);
const filepath = path.join(release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
await fs.writeFile(path.join(config.photoPath, filepath), res.body);
await knex('media').insert({
path: filepath,
mime: mimetype,
domain: 'releases',
target_id: releaseEntry.id,
role: 'trailer',
quality: release.trailer.quality || null,
});
}
async function storeReleases(releases = []) {
@@ -170,9 +200,21 @@ async function storeReleases(releases = []) {
})));
}
if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(path.join(config.photoPath, release.site.slug, releaseEntry.rows[0].id.toString()), { recursive: true });
}
if (release.photos && release.photos.length > 0) {
await storePhotos(release, releaseEntry.rows[0]);
}
if (release.poster) {
await storePoster(release, releaseEntry.rows[0]);
}
if (release.trailer) {
await storeTrailer(release, releaseEntry.rows[0]);
}
}, {
concurrency: 2,
});

4
src/releases.js
View File

@@ -28,7 +28,9 @@ async function curateRelease(release) {
actors,
director: release.director,
tags,
photos: media,
photos: media.filter(item => item.role === 'photo'),
poster: media.filter(item => item.role === 'poster')[0],
trailer: media.filter(item => item.role === 'trailer')[0],
rating: {
likes: release.likes,
dislikes: release.dislikes,

11
src/scrapers/kink.js
View File

@@ -18,6 +18,7 @@ function scrapeLatest(html, site) {
const shootId = href.split('/')[2];
const title = sceneLinkElement.text().trim();
const poster = $(element).find('.adimage').attr('src');
const photos = $(element).find('.rollover .roll-image').map((photoIndex, photoElement) => $(photoElement).attr('data-imagesrc')).toArray();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
@@ -36,6 +37,7 @@ function scrapeLatest(html, site) {
actors,
date,
photos,
poster,
rating: {
stars,
},
@@ -86,13 +88,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
actors,
description,
photos,
poster: trailerPoster,
trailer: {
video: {
default: trailerVideo,
sd: trailerVideo,
hd: trailerVideo.replace('480p', '720p'),
},
poster: trailerPoster,
src: trailerVideo,
quality: 480,
},
rating: {
stars,

28
src/scrapers/vixen.js
View File

@@ -15,18 +15,32 @@ function scrapeLatest(html, site) {
return scenes.map((scene) => {
const shootId = String(scene.newId);
const { title } = scene;
const {
title,
models: actors,
} = scene;
const url = `${site.url}${scene.targetUrl}`;
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
const actors = scene.models;
const stars = Number(scene.textRating) / 2;
// largest thumbnail. poster is the same image but bigger, too large for storage space efficiency
const poster = scene.images.listing.slice(-1)[0].src;
const trailer = scene.previews.listing.slice(-1)[0];
return {
url,
shootId,
title,
actors,
date,
poster,
trailer: {
src: trailer.src,
type: trailer.type,
quality: trailer.height,
},
rating: {
stars,
},
@@ -45,21 +59,21 @@ async function scrapeScene(html, url, site) {
const shootId = data.page.data[`${pathname}${search}`].data.video;
const scene = data.videos.find(video => video.newId === shootId);
// console.log(scene);
const {
title,
description,
models: actors,
totalRateVal: stars,
runLength: duration,
directorNames: director,
tags: rawTags,
} = scene;
const date = new Date(scene.releaseDate);
const rawTags = scene.tags;
const tags = await matchTags(rawTags);
const duration = scene.runLength;
const director = scene.directorNames;
return {
url,
shootId,