boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Using grid layout with thumbnails.

Browse Source
This commit is contained in:
ThePendulum
2019-05-08 05:50:13 +02:00
parent 8eb2dcfd89
commit e3558fc0c5
13 changed files with 412 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/fetch-releases.js
View File

@@ -1,8 +1,11 @@
'use strict';
const config = require('config');
const fs = require('fs-extra');
const path = require('path');
const Promise = require('bluebird');
const moment = require('moment');
const bhttp = require('bhttp');
const argv = require('./argv');
const knex = require('./knex');
@@ -75,9 +78,7 @@ async function findDuplicateReleases(latestReleases, _siteId) {
}
async function storeReleases(releases = []) {
return Promise.reduce(releases, async (acc, release) => {
await acc;
return Promise.map(releases, async (release) => {
const curatedRelease = {
site_id: release.site.id,
shoot_id: release.shootId || null,
@@ -115,7 +116,23 @@ async function storeReleases(releases = []) {
release_id: releaseEntry.rows[0].id,
})));
}
}, []);
if (release.thumbnails && release.thumbnails.length > 0) {
const thumbnailPath = path.join(config.thumbnailPath, release.site.id, releaseEntry.rows[0].id.toString());
await fs.mkdir(thumbnailPath, { recursive: true });
await Promise.map(release.thumbnails, async (thumbnailUrl, index) => {
const res = await bhttp.get(thumbnailUrl);
await fs.writeFile(path.join(thumbnailPath, `${index}.jpg`), res.body);
}, {
concurrency: 2,
});
}
}, {
concurrency: 2,
});
}
async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) {
@@ -151,7 +168,7 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page
async function fetchReleases() {
const sites = await accumulateIncludedSites();
const scenesPerSite = await Promise.all(sites.map(async (site) => {
const scenesPerSite = await Promise.map(sites, async (site) => {
const scraper = scrapers[site.id] || scrapers[site.network.id];
if (scraper) {
@@ -167,13 +184,18 @@ async function fetchReleases() {
if (argv.save) {
const finalReleases = argv.deep
? await Promise.all(newReleases.map(async (release) => {
? await Promise.map(newReleases, async (release) => {
if (release.url) {
return fetchScene(release.url);
const scene = await fetchScene(release.url, release);
return {
...release,
...scene,
};
}
return release;
}), {
}, {
concurrency: 2,
})
: newReleases;
@@ -204,7 +226,9 @@ async function fetchReleases() {
}
return [];
}));
}, {
concurrency: 2,
});
const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []);
const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA));

9
src/releases.js
View File

@@ -32,7 +32,11 @@ async function curateRelease(release) {
site: {
id: release.site_id,
name: release.site_name,
network: release.network_id,
},
network: {
id: release.network_id,
name: release.network_name,
url: release.network_url,
},
};
}
@@ -43,8 +47,9 @@ function curateReleases(releases) {
async function fetchReleases() {
const releases = await knex('releases')
.select('releases.*', 'sites.name as site_name')
.select('releases.*', 'sites.name as site_name', 'sites.network_id', 'networks.name as network_name', 'networks.url as network_url')
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.orderBy('date', 'desc')
.limit(100);

12
src/scrapers/julesjordan.js
View File

@@ -11,6 +11,10 @@ function scrapeLatest(html, site) {
const scenesElements = $('.update_details').toArray();
return scenesElements.map((element) => {
const thumbnailElement = $(element).find('a img.thumbs');
const thumbnailCount = Number(thumbnailElement.attr('cnt'));
const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined);
const sceneLinkElement = $(element).children('a').eq(1);
const url = sceneLinkElement.attr('href');
const title = sceneLinkElement.text();
@@ -32,6 +36,7 @@ function scrapeLatest(html, site) {
actors,
date,
site,
thumbnails,
};
});
}
@@ -41,6 +46,10 @@ function scrapeUpcoming(html, site) {
const scenesElements = $('#coming_soon_carousel').find('.table').toArray();
return scenesElements.map((element) => {
const thumbnailElement = $(element).find('a img.thumbs');
const thumbnailCount = Number(thumbnailElement.attr('cnt'));
const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined);
const shootId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0];
const details = $(element).find('.update_details_comingsoon')
@@ -66,8 +75,9 @@ function scrapeUpcoming(html, site) {
url: null,
shootId,
title,
actors,
date,
actors,
thumbnails,
rating: null,
site,
};

16
src/scrapers/kink.js
View File

@@ -18,6 +18,8 @@ function scrapeLatest(html, site) {
const shootId = href.split('/')[2];
const title = sceneLinkElement.text().trim();
const thumbnails = $(element).find('.rollover .roll-image').map((thumbnailIndex, thumbnailElement) => $(thumbnailElement).attr('data-imagesrc')).toArray();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const stars = $(element).find('.average-rating').attr('data-rating') / 10;
@@ -33,6 +35,7 @@ function scrapeLatest(html, site) {
title,
actors,
date,
thumbnails,
rating: {
stars,
},
@@ -49,6 +52,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const title = $('h1.shoot-title span.favorite-button').attr('data-title');
const actorsRaw = $('.shoot-info p.starring');
const thumbnails = $('.gallery .thumb img').map((thumbnailIndex, thumbnailElement) => `https://cdnp.kink.com${$(thumbnailElement).attr('data-image-file')}`).toArray();
const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url');
const trailerPoster = $('.player video#kink-player').attr('poster');
const date = moment.utc($(actorsRaw)
.prev()
.text()
@@ -78,6 +85,15 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
date,
actors,
description,
thumbnails,
trailer: {
video: {
default: trailerVideo,
sd: trailerVideo,
hd: trailerVideo.replace('480p', '720p'),
},
poster: trailerPoster,
},
rating: {
stars,
},

3
src/web/server.js
View File

@@ -12,6 +12,9 @@ function initServer() {
const app = express();
const router = Router();
app.use(express.static(config.thumbnailPath));
app.use(express.static('public'));
app.set('views', path.join(__dirname, '../../assets/views'));
app.set('view engine', 'jsx');
app.engine('jsx', createEngine());