Using grid layout with thumbnails.

This commit is contained in:
ThePendulum 2019-05-08 05:50:13 +02:00
parent 8eb2dcfd89
commit e3558fc0c5
13 changed files with 412 additions and 40 deletions

View File

@ -1,15 +1,21 @@
{
"root": true,
"parser": "babel-eslint",
"extends": "airbnb-base",
"extends": ["airbnb", "plugin:react/recommended"],
"plugins": ["react"],
"parserOptions": {
"sourceType": "script",
"ecmaFeatures": {
"jsx": true
}
},
"rules": {
"strict": 0,
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0,
"indent": ["error", 4],
"max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}]
"max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}],
"react/jsx-uses-vars": 2,
"react/jsx-indent": ["error", 4],
}
}

View File

@ -3,32 +3,88 @@
const React = require('react');
const moment = require('moment');
const Layout = require('./layout.jsx');
class Home extends React.Component {
render() {
return (
<table>
<tr>
<th>Date</th>
<th>ID</th>
<th>Shoot ID / Entry ID</th>
<th>Site</th>
<th>Title</th>
<th>Actors</th>
<th>Tags</th>
</tr>
<Layout>
<ul className="scenes">
{this.props.releases.map(release => (
<li key={release.id} className="scene">
<a
href={`/item/${release.id}`}
target="_blank"
>
<img
src={`/${release.site.id}/${release.id}/0.jpg`}
className="scene-thumbnail"
/>
</a>
{this.props.releases.map(release => (
<tr key={release.id}>
<td>{ moment(release.date).format('YYYY-MM-DD') }</td>
<td>{ release.id }</td>
<td>{ release.shootId || release.entryId }</td>
<td>{ release.site.name }</td>
<td>{ release.title }</td>
<td>{ release.actors.map(actor => actor.name).join(', ') }</td>
<td>{ release.tags.map(tag => tag.tag).join(', ') }</td>
</tr>
))}
</table>
<div className="scene-info">
<a
href={`/item/${release.id}`}
target="_blank"
rel="noopener noreferrer"
className="scene-row scene-link"
><h2 className="scene-title">{release.title}</h2></a>
<span className="scene-row">
<a
href={`/site/${release.site.id}`}
className="scene-site site-link"
>{release.site.name}</a>
<span>
<a
className="scene-network"
href={`/network/${release.network.id}`}
target="_blank"
rel="noopener noreferrer"
>{release.network.name}</a>
<a
href={release.url}
target="_blank"
rel="noopener noreferrer"
className="scene-date"
>{moment(release.date).format('YYYY-MM-DD')}</a>
</span>
</span>
<span className="scene-row">
<ul className="scene-actors nolist">{release.actors.map(actor =>
<li
key={actor.id}
className="scene-actor"
>
<a
href={`/actor/${actor.id}`}
className="actor-link"
>{actor.name}</a>
</li>
)}</ul>
</span>
<span className="scene-row">
<ul className="scene-tags nolist">{release.tags.map(tag =>
<li
key={tag.tag}
className="scene-tag"
>
<a
href={`/tag/${tag.tag}`}
className="tag-link"
>{tag.tag}</a>
</li>
)}</ul>
</span>
</div>
</li>
))}
</ul>
</Layout>
);
}
}

23
assets/views/layout.jsx Normal file
View File

@ -0,0 +1,23 @@
'use strict';
const React = require('react');
class Layout extends React.Component {
render() {
return (
<html lang="en">
<head>
<title>Traxxx</title>
<link href="/css/style.css" rel="stylesheet" />
</head>
<body>
{this.props.children}
</body>
</html>
);
}
}
module.exports = Layout;

View File

@ -81,6 +81,7 @@ module.exports = {
width: 30,
},
],
thumbnailPath: '/home/niels/Pictures/traxxx',
filename: {
dateFormat: 'DD-MM-YYYY',
actorsJoin: ', ',

124
package-lock.json generated
View File

@ -969,6 +969,16 @@
"sprintf-js": "~1.0.2"
}
},
"aria-query": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz",
"integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=",
"dev": true,
"requires": {
"ast-types-flow": "0.0.7",
"commander": "^2.11.0"
}
},
"arr-diff": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
@ -994,6 +1004,16 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"array-includes": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz",
"integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=",
"dev": true,
"requires": {
"define-properties": "^1.1.2",
"es-abstract": "^1.7.0"
}
},
"array-slice": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz",
@ -1022,6 +1042,12 @@
"resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
"integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c="
},
"ast-types-flow": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz",
"integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=",
"dev": true
},
"astral-regex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz",
@ -1054,6 +1080,15 @@
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
"integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
},
"axobject-query": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.0.2.tgz",
"integrity": "sha512-MCeek8ZH7hKyO1rWUbKNQBbl4l2eY0ntk7OGi+q0RlafrCnfPxC06WZA+uebCfmYp4mNU9jRBP1AhGyf8+W3ww==",
"dev": true,
"requires": {
"ast-types-flow": "0.0.7"
}
},
"babel-eslint": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.1.tgz",
@ -1740,6 +1775,12 @@
"lodash.get": "~4.4.2"
}
},
"damerau-levenshtein": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.4.tgz",
"integrity": "sha1-AxkcQyy27qFou3fzpV/9zLiXhRQ=",
"dev": true
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@ -2079,6 +2120,17 @@
}
}
},
"eslint-config-airbnb": {
"version": "17.1.0",
"resolved": "https://registry.npmjs.org/eslint-config-airbnb/-/eslint-config-airbnb-17.1.0.tgz",
"integrity": "sha512-R9jw28hFfEQnpPau01NO5K/JWMGLi6aymiF6RsnMURjTk+MqZKllCqGK/0tOvHkPi/NWSSOU2Ced/GX++YxLnw==",
"dev": true,
"requires": {
"eslint-config-airbnb-base": "^13.1.0",
"object.assign": "^4.1.0",
"object.entries": "^1.0.4"
}
},
"eslint-config-airbnb-base": {
"version": "13.1.0",
"resolved": "https://registry.npmjs.org/eslint-config-airbnb-base/-/eslint-config-airbnb-base-13.1.0.tgz",
@ -2140,6 +2192,57 @@
}
}
},
"eslint-plugin-jsx-a11y": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.1.tgz",
"integrity": "sha512-cjN2ObWrRz0TTw7vEcGQrx+YltMvZoOEx4hWU8eEERDnBIU00OTq7Vr+jA7DFKxiwLNv4tTh5Pq2GUNEa8b6+w==",
"dev": true,
"requires": {
"aria-query": "^3.0.0",
"array-includes": "^3.0.3",
"ast-types-flow": "^0.0.7",
"axobject-query": "^2.0.2",
"damerau-levenshtein": "^1.0.4",
"emoji-regex": "^7.0.2",
"has": "^1.0.3",
"jsx-ast-utils": "^2.0.1"
}
},
"eslint-plugin-react": {
"version": "7.13.0",
"resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.13.0.tgz",
"integrity": "sha512-uA5LrHylu8lW/eAH3bEQe9YdzpPaFd9yAJTwTi/i/BKTD7j6aQMKVAdGM/ML72zD6womuSK7EiGtMKuK06lWjQ==",
"dev": true,
"requires": {
"array-includes": "^3.0.3",
"doctrine": "^2.1.0",
"has": "^1.0.3",
"jsx-ast-utils": "^2.1.0",
"object.fromentries": "^2.0.0",
"prop-types": "^15.7.2",
"resolve": "^1.10.1"
},
"dependencies": {
"doctrine": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
"integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
"dev": true,
"requires": {
"esutils": "^2.0.2"
}
},
"resolve": {
"version": "1.10.1",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.1.tgz",
"integrity": "sha512-KuIe4mf++td/eFb6wkaPbMDnP6kObCaEtIDuHOUED6MNUo4K670KZUHuuvYPZDxNF0WVLw49n06M2m2dXphEzA==",
"dev": true,
"requires": {
"path-parse": "^1.0.6"
}
}
}
},
"eslint-restricted-globals": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/eslint-restricted-globals/-/eslint-restricted-globals-0.1.1.tgz",
@ -4031,6 +4134,15 @@
"verror": "1.10.0"
}
},
"jsx-ast-utils": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-2.1.0.tgz",
"integrity": "sha512-yDGDG2DS4JcqhA6blsuYbtsT09xL8AoLuUR2Gb5exrw7UEM19sBcOTq+YBBhrNbl0PUC4R4LnFu+dHg2HKeVvA==",
"dev": true,
"requires": {
"array-includes": "^3.0.3"
}
},
"keypress": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/keypress/-/keypress-0.2.1.tgz",
@ -4668,6 +4780,18 @@
"has": "^1.0.3"
}
},
"object.fromentries": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.0.tgz",
"integrity": "sha512-9iLiI6H083uiqUuvzyY6qrlmc/Gz8hLQFOcb/Ri/0xXFkSNS3ctV+CbE6yM2+AnkYfOB3dGjdzC0wrMLIhQICA==",
"dev": true,
"requires": {
"define-properties": "^1.1.2",
"es-abstract": "^1.11.0",
"function-bind": "^1.1.1",
"has": "^1.0.1"
}
},
"object.map": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz",

View File

@ -34,8 +34,10 @@
"babel-eslint": "^10.0.1",
"babel-preset-airbnb": "^3.2.0",
"eslint": "^5.15.0",
"eslint-config-airbnb-base": "^13.1.0",
"eslint-config-airbnb": "^17.1.0",
"eslint-plugin-import": "^2.16.0",
"eslint-plugin-jsx-a11y": "^6.2.1",
"eslint-plugin-react": "^7.13.0",
"eslint-watch": "^4.0.2"
},
"dependencies": {

102
public/css/style.css Normal file
View File

@ -0,0 +1,102 @@
body {
margin: 0;
}
.nolist {
list-style: none;
padding: 0;
margin: 0;
}
.nolist li {
display: inline-block;
}
.scenes {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
list-style: none;
padding: 0;
margin: 0;
}
.scene {
display: flex;
flex-direction: column;
box-sizing: border-box;
margin: .5rem;
height: 22rem;
box-shadow: 0 0 3px rgba(0, 0, 0, .5);
}
.scene-thumbnail {
width: 100%;
height: 200px;
object-fit: cover;
background-position: center;
background-size: cover;
}
.scene-row {
display: flex;
justify-content: space-between;
padding: .25rem .5rem;
}
.scene-info {
flex-grow: 1;
}
.scene-link {
text-decoration: none;
}
.scene-title {
color: #000;
margin: 0;
font-size: 1rem;
}
.scene-site {
font-weight: bold;
font-size: .8rem;
}
.scene-date {
color: #555;
font-size: .8rem;
}
.scene-network {
color: #555;
margin: 0 .25rem 0 0;
font-size: .8rem;
}
.scene-tags {
white-space: nowrap;
overflow: hidden;
}
.scene-actor,
.scene-tag {
margin: 0 .25rem 0 0;
}
.scene-tag {
font-size: .75rem;
}
.scene-actor:not(:last-of-type)::after,
.scene-tag:not(:last-of-type):after {
content: ",";
}
.site-link {
color: #000;
}
.actor-link,
.tag-link {
color: #000;
}

View File

@ -882,8 +882,8 @@ exports.seed = knex => Promise.resolve()
network_id: 'kink',
},
{
id: 'devinebitches',
name: 'Devine Bitches',
id: 'divinebitches',
name: 'Divine Bitches',
url: 'https://www.kink.com/channel/divinebitches',
description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.',
network_id: 'kink',
@ -1008,7 +1008,7 @@ exports.seed = knex => Promise.resolve()
network_id: 'kink',
},
{
id: 'tspussyhunts',
id: 'tspussyhunters',
name: 'TS Pussy Hunters',
url: 'https://www.kink.com/channel/tspussyhunters',
description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.',

View File

@ -1,8 +1,11 @@
'use strict';
const config = require('config');
const fs = require('fs-extra');
const path = require('path');
const Promise = require('bluebird');
const moment = require('moment');
const bhttp = require('bhttp');
const argv = require('./argv');
const knex = require('./knex');
@ -75,9 +78,7 @@ async function findDuplicateReleases(latestReleases, _siteId) {
}
async function storeReleases(releases = []) {
return Promise.reduce(releases, async (acc, release) => {
await acc;
return Promise.map(releases, async (release) => {
const curatedRelease = {
site_id: release.site.id,
shoot_id: release.shootId || null,
@ -115,7 +116,23 @@ async function storeReleases(releases = []) {
release_id: releaseEntry.rows[0].id,
})));
}
}, []);
if (release.thumbnails && release.thumbnails.length > 0) {
const thumbnailPath = path.join(config.thumbnailPath, release.site.id, releaseEntry.rows[0].id.toString());
await fs.mkdir(thumbnailPath, { recursive: true });
await Promise.map(release.thumbnails, async (thumbnailUrl, index) => {
const res = await bhttp.get(thumbnailUrl);
await fs.writeFile(path.join(thumbnailPath, `${index}.jpg`), res.body);
}, {
concurrency: 2,
});
}
}, {
concurrency: 2,
});
}
async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) {
@ -151,7 +168,7 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page
async function fetchReleases() {
const sites = await accumulateIncludedSites();
const scenesPerSite = await Promise.all(sites.map(async (site) => {
const scenesPerSite = await Promise.map(sites, async (site) => {
const scraper = scrapers[site.id] || scrapers[site.network.id];
if (scraper) {
@ -167,13 +184,18 @@ async function fetchReleases() {
if (argv.save) {
const finalReleases = argv.deep
? await Promise.all(newReleases.map(async (release) => {
? await Promise.map(newReleases, async (release) => {
if (release.url) {
return fetchScene(release.url);
const scene = await fetchScene(release.url, release);
return {
...release,
...scene,
};
}
return release;
}), {
}, {
concurrency: 2,
})
: newReleases;
@ -204,7 +226,9 @@ async function fetchReleases() {
}
return [];
}));
}, {
concurrency: 2,
});
const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []);
const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA));

View File

@ -32,7 +32,11 @@ async function curateRelease(release) {
site: {
id: release.site_id,
name: release.site_name,
network: release.network_id,
},
network: {
id: release.network_id,
name: release.network_name,
url: release.network_url,
},
};
}
@ -43,8 +47,9 @@ function curateReleases(releases) {
async function fetchReleases() {
const releases = await knex('releases')
.select('releases.*', 'sites.name as site_name')
.select('releases.*', 'sites.name as site_name', 'sites.network_id', 'networks.name as network_name', 'networks.url as network_url')
.leftJoin('sites', 'releases.site_id', 'sites.id')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.orderBy('date', 'desc')
.limit(100);

View File

@ -11,6 +11,10 @@ function scrapeLatest(html, site) {
const scenesElements = $('.update_details').toArray();
return scenesElements.map((element) => {
const thumbnailElement = $(element).find('a img.thumbs');
const thumbnailCount = Number(thumbnailElement.attr('cnt'));
const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined);
const sceneLinkElement = $(element).children('a').eq(1);
const url = sceneLinkElement.attr('href');
const title = sceneLinkElement.text();
@ -32,6 +36,7 @@ function scrapeLatest(html, site) {
actors,
date,
site,
thumbnails,
};
});
}
@ -41,6 +46,10 @@ function scrapeUpcoming(html, site) {
const scenesElements = $('#coming_soon_carousel').find('.table').toArray();
return scenesElements.map((element) => {
const thumbnailElement = $(element).find('a img.thumbs');
const thumbnailCount = Number(thumbnailElement.attr('cnt'));
const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined);
const shootId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0];
const details = $(element).find('.update_details_comingsoon')
@ -66,8 +75,9 @@ function scrapeUpcoming(html, site) {
url: null,
shootId,
title,
actors,
date,
actors,
thumbnails,
rating: null,
site,
};

View File

@ -18,6 +18,8 @@ function scrapeLatest(html, site) {
const shootId = href.split('/')[2];
const title = sceneLinkElement.text().trim();
const thumbnails = $(element).find('.rollover .roll-image').map((thumbnailIndex, thumbnailElement) => $(thumbnailElement).attr('data-imagesrc')).toArray();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const stars = $(element).find('.average-rating').attr('data-rating') / 10;
@ -33,6 +35,7 @@ function scrapeLatest(html, site) {
title,
actors,
date,
thumbnails,
rating: {
stars,
},
@ -49,6 +52,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const title = $('h1.shoot-title span.favorite-button').attr('data-title');
const actorsRaw = $('.shoot-info p.starring');
const thumbnails = $('.gallery .thumb img').map((thumbnailIndex, thumbnailElement) => `https://cdnp.kink.com${$(thumbnailElement).attr('data-image-file')}`).toArray();
const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url');
const trailerPoster = $('.player video#kink-player').attr('poster');
const date = moment.utc($(actorsRaw)
.prev()
.text()
@ -78,6 +85,15 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
date,
actors,
description,
thumbnails,
trailer: {
video: {
default: trailerVideo,
sd: trailerVideo,
hd: trailerVideo.replace('480p', '720p'),
},
poster: trailerPoster,
},
rating: {
stars,
},

View File

@ -12,6 +12,9 @@ function initServer() {
const app = express();
const router = Router();
app.use(express.static(config.thumbnailPath));
app.use(express.static('public'));
app.set('views', path.join(__dirname, '../../assets/views'));
app.set('view engine', 'jsx');
app.engine('jsx', createEngine());