From e3558fc0c56d74d4ce596e5808af6f25177fa2de Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Wed, 8 May 2019 05:50:13 +0200 Subject: [PATCH] Using grid layout with thumbnails. --- .eslintrc | 10 ++- assets/views/home.jsx | 100 ++++++++++++++++++++++------- assets/views/layout.jsx | 23 +++++++ config/default.js | 1 + package-lock.json | 124 ++++++++++++++++++++++++++++++++++++ package.json | 4 +- public/css/style.css | 102 +++++++++++++++++++++++++++++ seeds/01_sites.js | 6 +- src/fetch-releases.js | 42 +++++++++--- src/releases.js | 9 ++- src/scrapers/julesjordan.js | 12 +++- src/scrapers/kink.js | 16 +++++ src/web/server.js | 3 + 13 files changed, 412 insertions(+), 40 deletions(-) create mode 100644 assets/views/layout.jsx create mode 100644 public/css/style.css diff --git a/.eslintrc b/.eslintrc index 9f9fb622e..b4941ed3d 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,15 +1,21 @@ { "root": true, "parser": "babel-eslint", - "extends": "airbnb-base", + "extends": ["airbnb", "plugin:react/recommended"], + "plugins": ["react"], "parserOptions": { "sourceType": "script", + "ecmaFeatures": { + "jsx": true + } }, "rules": { "strict": 0, "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-console": 0, "indent": ["error", 4], - "max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}] + "max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}], + "react/jsx-uses-vars": 2, + "react/jsx-indent": ["error", 4], } } diff --git a/assets/views/home.jsx b/assets/views/home.jsx index 14ab61cec..6b308b604 100644 --- a/assets/views/home.jsx +++ b/assets/views/home.jsx @@ -3,32 +3,88 @@ const React = require('react'); const moment = require('moment'); +const Layout = require('./layout.jsx'); + class Home extends React.Component { render() { return ( - - - - - - - - - - + + - - - - - - - - - ))} -
DateIDShoot ID / Entry IDSiteTitleActorsTags
{ moment(release.date).format('YYYY-MM-DD') }{ release.id }{ release.shootId || release.entryId }{ release.site.name }{ release.title }{ release.actors.map(actor => actor.name).join(', ') }{ release.tags.map(tag => tag.tag).join(', ') }
+
+

{release.title}

 + + + {release.site.name} + + + {release.network.name} + + {moment(release.date).format('YYYY-MM-DD')} + + + + + + + + +
    {release.tags.map(tag => +
  • + {tag.tag} +
    • + )}
+ +
+ + ))} + + ); } } diff --git a/assets/views/layout.jsx b/assets/views/layout.jsx new file mode 100644 index 000000000..6d4e1211d --- /dev/null +++ b/assets/views/layout.jsx @@ -0,0 +1,23 @@ +'use strict'; + +const React = require('react'); + +class Layout extends React.Component { + render() { + return ( + + + Traxxx + + + + + + {this.props.children} + + + ); + } +} + +module.exports = Layout; diff --git a/config/default.js b/config/default.js index c10eedeff..8bedd3051 100644 --- a/config/default.js +++ b/config/default.js @@ -81,6 +81,7 @@ module.exports = { width: 30, }, ], + thumbnailPath: '/home/niels/Pictures/traxxx', filename: { dateFormat: 'DD-MM-YYYY', actorsJoin: ', ', diff --git a/package-lock.json b/package-lock.json index 94b0192eb..bcf16787b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -969,6 +969,16 @@ "sprintf-js": "~1.0.2" } }, + "aria-query": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz", + "integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=", + "dev": true, + "requires": { + "ast-types-flow": "0.0.7", + "commander": "^2.11.0" + } + }, "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", @@ -994,6 +1004,16 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "array-includes": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz", + "integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "es-abstract": "^1.7.0" + } + }, "array-slice": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz", @@ -1022,6 +1042,12 @@ "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, + "ast-types-flow": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", + "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=", + "dev": true + }, "astral-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", @@ -1054,6 +1080,15 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, + "axobject-query": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.0.2.tgz", + "integrity": "sha512-MCeek8ZH7hKyO1rWUbKNQBbl4l2eY0ntk7OGi+q0RlafrCnfPxC06WZA+uebCfmYp4mNU9jRBP1AhGyf8+W3ww==", + "dev": true, + "requires": { + "ast-types-flow": "0.0.7" + } + }, "babel-eslint": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.1.tgz", @@ -1740,6 +1775,12 @@ "lodash.get": "~4.4.2" } }, + "damerau-levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.4.tgz", + "integrity": "sha1-AxkcQyy27qFou3fzpV/9zLiXhRQ=", + "dev": true + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -2079,6 +2120,17 @@ } } }, + "eslint-config-airbnb": { + "version": "17.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-airbnb/-/eslint-config-airbnb-17.1.0.tgz", + "integrity": "sha512-R9jw28hFfEQnpPau01NO5K/JWMGLi6aymiF6RsnMURjTk+MqZKllCqGK/0tOvHkPi/NWSSOU2Ced/GX++YxLnw==", + "dev": true, + "requires": { + "eslint-config-airbnb-base": "^13.1.0", + "object.assign": "^4.1.0", + "object.entries": "^1.0.4" + } + }, "eslint-config-airbnb-base": { "version": "13.1.0", "resolved": "https://registry.npmjs.org/eslint-config-airbnb-base/-/eslint-config-airbnb-base-13.1.0.tgz", @@ -2140,6 +2192,57 @@ } } }, + "eslint-plugin-jsx-a11y": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.1.tgz", + "integrity": "sha512-cjN2ObWrRz0TTw7vEcGQrx+YltMvZoOEx4hWU8eEERDnBIU00OTq7Vr+jA7DFKxiwLNv4tTh5Pq2GUNEa8b6+w==", + "dev": true, + "requires": { + "aria-query": "^3.0.0", + "array-includes": "^3.0.3", + "ast-types-flow": "^0.0.7", + "axobject-query": "^2.0.2", + "damerau-levenshtein": "^1.0.4", + "emoji-regex": "^7.0.2", + "has": "^1.0.3", + "jsx-ast-utils": "^2.0.1" + } + }, + "eslint-plugin-react": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.13.0.tgz", + "integrity": "sha512-uA5LrHylu8lW/eAH3bEQe9YdzpPaFd9yAJTwTi/i/BKTD7j6aQMKVAdGM/ML72zD6womuSK7EiGtMKuK06lWjQ==", + "dev": true, + "requires": { + "array-includes": "^3.0.3", + "doctrine": "^2.1.0", + "has": "^1.0.3", + "jsx-ast-utils": "^2.1.0", + "object.fromentries": "^2.0.0", + "prop-types": "^15.7.2", + "resolve": "^1.10.1" + }, + "dependencies": { + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "resolve": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.1.tgz", + "integrity": "sha512-KuIe4mf++td/eFb6wkaPbMDnP6kObCaEtIDuHOUED6MNUo4K670KZUHuuvYPZDxNF0WVLw49n06M2m2dXphEzA==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, "eslint-restricted-globals": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/eslint-restricted-globals/-/eslint-restricted-globals-0.1.1.tgz", @@ -4031,6 +4134,15 @@ "verror": "1.10.0" } }, + "jsx-ast-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-2.1.0.tgz", + "integrity": "sha512-yDGDG2DS4JcqhA6blsuYbtsT09xL8AoLuUR2Gb5exrw7UEM19sBcOTq+YBBhrNbl0PUC4R4LnFu+dHg2HKeVvA==", + "dev": true, + "requires": { + "array-includes": "^3.0.3" + } + }, "keypress": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/keypress/-/keypress-0.2.1.tgz", @@ -4668,6 +4780,18 @@ "has": "^1.0.3" } }, + "object.fromentries": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.0.tgz", + "integrity": "sha512-9iLiI6H083uiqUuvzyY6qrlmc/Gz8hLQFOcb/Ri/0xXFkSNS3ctV+CbE6yM2+AnkYfOB3dGjdzC0wrMLIhQICA==", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "es-abstract": "^1.11.0", + "function-bind": "^1.1.1", + "has": "^1.0.1" + } + }, "object.map": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz", diff --git a/package.json b/package.json index 978a1a181..ce82dce64 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,10 @@ "babel-eslint": "^10.0.1", "babel-preset-airbnb": "^3.2.0", "eslint": "^5.15.0", - "eslint-config-airbnb-base": "^13.1.0", + "eslint-config-airbnb": "^17.1.0", "eslint-plugin-import": "^2.16.0", + "eslint-plugin-jsx-a11y": "^6.2.1", + "eslint-plugin-react": "^7.13.0", "eslint-watch": "^4.0.2" }, "dependencies": { diff --git a/public/css/style.css b/public/css/style.css new file mode 100644 index 000000000..f9edc82d9 --- /dev/null +++ b/public/css/style.css @@ -0,0 +1,102 @@ +body { + margin: 0; +} + +.nolist { + list-style: none; + padding: 0; + margin: 0; +} + +.nolist li { + display: inline-block; +} + +.scenes { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr)); + list-style: none; + padding: 0; + margin: 0; +} + +.scene { + display: flex; + flex-direction: column; + box-sizing: border-box; + margin: .5rem; + height: 22rem; + box-shadow: 0 0 3px rgba(0, 0, 0, .5); +} + +.scene-thumbnail { + width: 100%; + height: 200px; + object-fit: cover; + background-position: center; + background-size: cover; +} + +.scene-row { + display: flex; + justify-content: space-between; + padding: .25rem .5rem; +} + +.scene-info { + flex-grow: 1; +} + +.scene-link { + text-decoration: none; +} + +.scene-title { + color: #000; + margin: 0; + font-size: 1rem; +} + +.scene-site { + font-weight: bold; + font-size: .8rem; +} + +.scene-date { + color: #555; + font-size: .8rem; +} + +.scene-network { + color: #555; + margin: 0 .25rem 0 0; + font-size: .8rem; +} + +.scene-tags { + white-space: nowrap; + overflow: hidden; +} + +.scene-actor, +.scene-tag { + margin: 0 .25rem 0 0; +} + +.scene-tag { + font-size: .75rem; +} + +.scene-actor:not(:last-of-type)::after, +.scene-tag:not(:last-of-type):after { + content: ","; +} + +.site-link { + color: #000; +} + +.actor-link, +.tag-link { + color: #000; +} diff --git a/seeds/01_sites.js b/seeds/01_sites.js index dbaa037ac..317928715 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -882,8 +882,8 @@ exports.seed = knex => Promise.resolve() network_id: 'kink', }, { - id: 'devinebitches', - name: 'Devine Bitches', + id: 'divinebitches', + name: 'Divine Bitches', url: 'https://www.kink.com/channel/divinebitches', description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.', network_id: 'kink', @@ -1008,7 +1008,7 @@ exports.seed = knex => Promise.resolve() network_id: 'kink', }, { - id: 'tspussyhunts', + id: 'tspussyhunters', name: 'TS Pussy Hunters', url: 'https://www.kink.com/channel/tspussyhunters', description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.', diff --git a/src/fetch-releases.js b/src/fetch-releases.js index 7a901b12c..53ce21409 100644 --- a/src/fetch-releases.js +++ b/src/fetch-releases.js @@ -1,8 +1,11 @@ 'use strict'; const config = require('config'); +const fs = require('fs-extra'); +const path = require('path'); const Promise = require('bluebird'); const moment = require('moment'); +const bhttp = require('bhttp'); const argv = require('./argv'); const knex = require('./knex'); @@ -75,9 +78,7 @@ async function findDuplicateReleases(latestReleases, _siteId) { } async function storeReleases(releases = []) { - return Promise.reduce(releases, async (acc, release) => { - await acc; - + return Promise.map(releases, async (release) => { const curatedRelease = { site_id: release.site.id, shoot_id: release.shootId || null, @@ -115,7 +116,23 @@ async function storeReleases(releases = []) { release_id: releaseEntry.rows[0].id, }))); } - }, []); + + if (release.thumbnails && release.thumbnails.length > 0) { + const thumbnailPath = path.join(config.thumbnailPath, release.site.id, releaseEntry.rows[0].id.toString()); + + await fs.mkdir(thumbnailPath, { recursive: true }); + + await Promise.map(release.thumbnails, async (thumbnailUrl, index) => { + const res = await bhttp.get(thumbnailUrl); + + await fs.writeFile(path.join(thumbnailPath, `${index}.jpg`), res.body); + }, { + concurrency: 2, + }); + } + }, { + concurrency: 2, + }); } async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) { @@ -151,7 +168,7 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page async function fetchReleases() { const sites = await accumulateIncludedSites(); - const scenesPerSite = await Promise.all(sites.map(async (site) => { + const scenesPerSite = await Promise.map(sites, async (site) => { const scraper = scrapers[site.id] || scrapers[site.network.id]; if (scraper) { @@ -167,13 +184,18 @@ async function fetchReleases() { if (argv.save) { const finalReleases = argv.deep - ? await Promise.all(newReleases.map(async (release) => { + ? await Promise.map(newReleases, async (release) => { if (release.url) { - return fetchScene(release.url); + const scene = await fetchScene(release.url, release); + + return { + ...release, + ...scene, + }; } return release; - }), { + }, { concurrency: 2, }) : newReleases; @@ -204,7 +226,9 @@ async function fetchReleases() { } return []; - })); + }, { + concurrency: 2, + }); const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []); const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA)); diff --git a/src/releases.js b/src/releases.js index 490d33b0e..ea4defe1c 100644 --- a/src/releases.js +++ b/src/releases.js @@ -32,7 +32,11 @@ async function curateRelease(release) { site: { id: release.site_id, name: release.site_name, - network: release.network_id, + }, + network: { + id: release.network_id, + name: release.network_name, + url: release.network_url, }, }; } @@ -43,8 +47,9 @@ function curateReleases(releases) { async function fetchReleases() { const releases = await knex('releases') - .select('releases.*', 'sites.name as site_name') + .select('releases.*', 'sites.name as site_name', 'sites.network_id', 'networks.name as network_name', 'networks.url as network_url') .leftJoin('sites', 'releases.site_id', 'sites.id') + .leftJoin('networks', 'sites.network_id', 'networks.id') .orderBy('date', 'desc') .limit(100); diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index f142bdb6a..5f4844913 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -11,6 +11,10 @@ function scrapeLatest(html, site) { const scenesElements = $('.update_details').toArray(); return scenesElements.map((element) => { + const thumbnailElement = $(element).find('a img.thumbs'); + const thumbnailCount = Number(thumbnailElement.attr('cnt')); + const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined); + const sceneLinkElement = $(element).children('a').eq(1); const url = sceneLinkElement.attr('href'); const title = sceneLinkElement.text(); @@ -32,6 +36,7 @@ function scrapeLatest(html, site) { actors, date, site, + thumbnails, }; }); } @@ -41,6 +46,10 @@ function scrapeUpcoming(html, site) { const scenesElements = $('#coming_soon_carousel').find('.table').toArray(); return scenesElements.map((element) => { + const thumbnailElement = $(element).find('a img.thumbs'); + const thumbnailCount = Number(thumbnailElement.attr('cnt')); + const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined); + const shootId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0]; const details = $(element).find('.update_details_comingsoon') @@ -66,8 +75,9 @@ function scrapeUpcoming(html, site) { url: null, shootId, title, - actors, date, + actors, + thumbnails, rating: null, site, }; diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js index ef2e56c89..24fbed0a8 100644 --- a/src/scrapers/kink.js +++ b/src/scrapers/kink.js @@ -18,6 +18,8 @@ function scrapeLatest(html, site) { const shootId = href.split('/')[2]; const title = sceneLinkElement.text().trim(); + const thumbnails = $(element).find('.rollover .roll-image').map((thumbnailIndex, thumbnailElement) => $(thumbnailElement).attr('data-imagesrc')).toArray(); + const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate(); const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const stars = $(element).find('.average-rating').attr('data-rating') / 10; @@ -33,6 +35,7 @@ function scrapeLatest(html, site) { title, actors, date, + thumbnails, rating: { stars, }, @@ -49,6 +52,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) { const title = $('h1.shoot-title span.favorite-button').attr('data-title'); const actorsRaw = $('.shoot-info p.starring'); + const thumbnails = $('.gallery .thumb img').map((thumbnailIndex, thumbnailElement) => `https://cdnp.kink.com${$(thumbnailElement).attr('data-image-file')}`).toArray(); + const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url'); + const trailerPoster = $('.player video#kink-player').attr('poster'); + const date = moment.utc($(actorsRaw) .prev() .text() @@ -78,6 +85,15 @@ async function scrapeScene(html, url, shootId, ratingRes, site) { date, actors, description, + thumbnails, + trailer: { + video: { + default: trailerVideo, + sd: trailerVideo, + hd: trailerVideo.replace('480p', '720p'), + }, + poster: trailerPoster, + }, rating: { stars, }, diff --git a/src/web/server.js b/src/web/server.js index a86b91dea..c7145decd 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -12,6 +12,9 @@ function initServer() { const app = express(); const router = Router(); + app.use(express.static(config.thumbnailPath)); + app.use(express.static('public')); + app.set('views', path.join(__dirname, '../../assets/views')); app.set('view engine', 'jsx'); app.engine('jsx', createEngine());