From e3558fc0c56d74d4ce596e5808af6f25177fa2de Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Wed, 8 May 2019 05:50:13 +0200 Subject: [PATCH] Using grid layout with thumbnails. --- .eslintrc | 10 ++- assets/views/home.jsx | 100 ++++++++++++++++++++++------- assets/views/layout.jsx | 23 +++++++ config/default.js | 1 + package-lock.json | 124 ++++++++++++++++++++++++++++++++++++ package.json | 4 +- public/css/style.css | 102 +++++++++++++++++++++++++++++ seeds/01_sites.js | 6 +- src/fetch-releases.js | 42 +++++++++--- src/releases.js | 9 ++- src/scrapers/julesjordan.js | 12 +++- src/scrapers/kink.js | 16 +++++ src/web/server.js | 3 + 13 files changed, 412 insertions(+), 40 deletions(-) create mode 100644 assets/views/layout.jsx create mode 100644 public/css/style.css diff --git a/.eslintrc b/.eslintrc index 9f9fb622..b4941ed3 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,15 +1,21 @@ { "root": true, "parser": "babel-eslint", - "extends": "airbnb-base", + "extends": ["airbnb", "plugin:react/recommended"], + "plugins": ["react"], "parserOptions": { "sourceType": "script", + "ecmaFeatures": { + "jsx": true + } }, "rules": { "strict": 0, "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-console": 0, "indent": ["error", 4], - "max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}] + "max-len": [2, {"code": 200, "tabWidth": 4, "ignoreUrls": true}], + "react/jsx-uses-vars": 2, + "react/jsx-indent": ["error", 4], } } diff --git a/assets/views/home.jsx b/assets/views/home.jsx index 14ab61ce..6b308b60 100644 --- a/assets/views/home.jsx +++ b/assets/views/home.jsx @@ -3,32 +3,88 @@ const React = require('react'); const moment = require('moment'); +const Layout = require('./layout.jsx'); + class Home extends React.Component { render() { return ( - - - - - - - - - - + + - - - - - - - - - ))} -
DateIDShoot ID / Entry IDSiteTitleActorsTags
{ moment(release.date).format('YYYY-MM-DD') }{ release.id }{ release.shootId || release.entryId }{ release.site.name }{ release.title }{ release.actors.map(actor => actor.name).join(', ') }{ release.tags.map(tag => tag.tag).join(', ') }
+
+

{release.title}

 + + + {release.site.name} + + + {release.network.name} + + {moment(release.date).format('YYYY-MM-DD')} + + + + + + + + +
    {release.tags.map(tag => +
  • + {tag.tag} +
    • + )}
+ +
+ + ))} + + ); } } diff --git a/assets/views/layout.jsx b/assets/views/layout.jsx new file mode 100644 index 00000000..6d4e1211 --- /dev/null +++ b/assets/views/layout.jsx @@ -0,0 +1,23 @@ +'use strict'; + +const React = require('react'); + +class Layout extends React.Component { + render() { + return ( + + + Traxxx + + + + + + {this.props.children} + + + ); + } +} + +module.exports = Layout; diff --git a/config/default.js b/config/default.js index c10eedef..8bedd305 100644 --- a/config/default.js +++ b/config/default.js @@ -81,6 +81,7 @@ module.exports = { width: 30, }, ], + thumbnailPath: '/home/niels/Pictures/traxxx', filename: { dateFormat: 'DD-MM-YYYY', actorsJoin: ', ', diff --git a/package-lock.json b/package-lock.json index 94b0192e..bcf16787 100644 --- a/package-lock.json +++ b/package-lock.json @@ -969,6 +969,16 @@ "sprintf-js": "~1.0.2" } }, + "aria-query": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz", + "integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=", + "dev": true, + "requires": { + "ast-types-flow": "0.0.7", + "commander": "^2.11.0" + } + }, "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", @@ -994,6 +1004,16 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "array-includes": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz", + "integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "es-abstract": "^1.7.0" + } + }, "array-slice": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz", @@ -1022,6 +1042,12 @@ "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, + "ast-types-flow": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", + "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=", + "dev": true + }, "astral-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", @@ -1054,6 +1080,15 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, + "axobject-query": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.0.2.tgz", + "integrity": "sha512-MCeek8ZH7hKyO1rWUbKNQBbl4l2eY0ntk7OGi+q0RlafrCnfPxC06WZA+uebCfmYp4mNU9jRBP1AhGyf8+W3ww==", + "dev": true, + "requires": { + "ast-types-flow": "0.0.7" + } + }, "babel-eslint": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.1.tgz", @@ -1740,6 +1775,12 @@ "lodash.get": "~4.4.2" } }, + "damerau-levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.4.tgz", + "integrity": "sha1-AxkcQyy27qFou3fzpV/9zLiXhRQ=", + "dev": true + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -2079,6 +2120,17 @@ } } }, + "eslint-config-airbnb": { + "version": "17.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-airbnb/-/eslint-config-airbnb-17.1.0.tgz", + "integrity": "sha512-R9jw28hFfEQnpPau01NO5K/JWMGLi6aymiF6RsnMURjTk+MqZKllCqGK/0tOvHkPi/NWSSOU2Ced/GX++YxLnw==", + "dev": true, + "requires": { + "eslint-config-airbnb-base": "^13.1.0", + "object.assign": "^4.1.0", + "object.entries": "^1.0.4" + } + }, "eslint-config-airbnb-base": { "version": "13.1.0", "resolved": "https://registry.npmjs.org/eslint-config-airbnb-base/-/eslint-config-airbnb-base-13.1.0.tgz", @@ -2140,6 +2192,57 @@ } } }, + "eslint-plugin-jsx-a11y": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.1.tgz", + "integrity": "sha512-cjN2ObWrRz0TTw7vEcGQrx+YltMvZoOEx4hWU8eEERDnBIU00OTq7Vr+jA7DFKxiwLNv4tTh5Pq2GUNEa8b6+w==", + "dev": true, + "requires": { + "aria-query": "^3.0.0", + "array-includes": "^3.0.3", + "ast-types-flow": "^0.0.7", + "axobject-query": "^2.0.2", + "damerau-levenshtein": "^1.0.4", + "emoji-regex": "^7.0.2", + "has": "^1.0.3", + "jsx-ast-utils": "^2.0.1" + } + }, + "eslint-plugin-react": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.13.0.tgz", + "integrity": "sha512-uA5LrHylu8lW/eAH3bEQe9YdzpPaFd9yAJTwTi/i/BKTD7j6aQMKVAdGM/ML72zD6womuSK7EiGtMKuK06lWjQ==", + "dev": true, + "requires": { + "array-includes": "^3.0.3", + "doctrine": "^2.1.0", + "has": "^1.0.3", + "jsx-ast-utils": "^2.1.0", + "object.fromentries": "^2.0.0", + "prop-types": "^15.7.2", + "resolve": "^1.10.1" + }, + "dependencies": { + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "resolve": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.1.tgz", + "integrity": "sha512-KuIe4mf++td/eFb6wkaPbMDnP6kObCaEtIDuHOUED6MNUo4K670KZUHuuvYPZDxNF0WVLw49n06M2m2dXphEzA==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, "eslint-restricted-globals": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/eslint-restricted-globals/-/eslint-restricted-globals-0.1.1.tgz", @@ -4031,6 +4134,15 @@ "verror": "1.10.0" } }, + "jsx-ast-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-2.1.0.tgz", + "integrity": "sha512-yDGDG2DS4JcqhA6blsuYbtsT09xL8AoLuUR2Gb5exrw7UEM19sBcOTq+YBBhrNbl0PUC4R4LnFu+dHg2HKeVvA==", + "dev": true, + "requires": { + "array-includes": "^3.0.3" + } + }, "keypress": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/keypress/-/keypress-0.2.1.tgz", @@ -4668,6 +4780,18 @@ "has": "^1.0.3" } }, + "object.fromentries": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.0.tgz", + "integrity": "sha512-9iLiI6H083uiqUuvzyY6qrlmc/Gz8hLQFOcb/Ri/0xXFkSNS3ctV+CbE6yM2+AnkYfOB3dGjdzC0wrMLIhQICA==", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "es-abstract": "^1.11.0", + "function-bind": "^1.1.1", + "has": "^1.0.1" + } + }, "object.map": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object.map/-/object.map-1.0.1.tgz", diff --git a/package.json b/package.json index 978a1a18..ce82dce6 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,10 @@ "babel-eslint": "^10.0.1", "babel-preset-airbnb": "^3.2.0", "eslint": "^5.15.0", - "eslint-config-airbnb-base": "^13.1.0", + "eslint-config-airbnb": "^17.1.0", "eslint-plugin-import": "^2.16.0", + "eslint-plugin-jsx-a11y": "^6.2.1", + "eslint-plugin-react": "^7.13.0", "eslint-watch": "^4.0.2" }, "dependencies": { diff --git a/public/css/style.css b/public/css/style.css new file mode 100644 index 00000000..f9edc82d --- /dev/null +++ b/public/css/style.css @@ -0,0 +1,102 @@ +body { + margin: 0; +} + +.nolist { + list-style: none; + padding: 0; + margin: 0; +} + +.nolist li { + display: inline-block; +} + +.scenes { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr)); + list-style: none; + padding: 0; + margin: 0; +} + +.scene { + display: flex; + flex-direction: column; + box-sizing: border-box; + margin: .5rem; + height: 22rem; + box-shadow: 0 0 3px rgba(0, 0, 0, .5); +} + +.scene-thumbnail { + width: 100%; + height: 200px; + object-fit: cover; + background-position: center; + background-size: cover; +} + +.scene-row { + display: flex; + justify-content: space-between; + padding: .25rem .5rem; +} + +.scene-info { + flex-grow: 1; +} + +.scene-link { + text-decoration: none; +} + +.scene-title { + color: #000; + margin: 0; + font-size: 1rem; +} + +.scene-site { + font-weight: bold; + font-size: .8rem; +} + +.scene-date { + color: #555; + font-size: .8rem; +} + +.scene-network { + color: #555; + margin: 0 .25rem 0 0; + font-size: .8rem; +} + +.scene-tags { + white-space: nowrap; + overflow: hidden; +} + +.scene-actor, +.scene-tag { + margin: 0 .25rem 0 0; +} + +.scene-tag { + font-size: .75rem; +} + +.scene-actor:not(:last-of-type)::after, +.scene-tag:not(:last-of-type):after { + content: ","; +} + +.site-link { + color: #000; +} + +.actor-link, +.tag-link { + color: #000; +} diff --git a/seeds/01_sites.js b/seeds/01_sites.js index dbaa037a..31792871 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -882,8 +882,8 @@ exports.seed = knex => Promise.resolve() network_id: 'kink', }, { - id: 'devinebitches', - name: 'Devine Bitches', + id: 'divinebitches', + name: 'Divine Bitches', url: 'https://www.kink.com/channel/divinebitches', description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.', network_id: 'kink', @@ -1008,7 +1008,7 @@ exports.seed = knex => Promise.resolve() network_id: 'kink', }, { - id: 'tspussyhunts', + id: 'tspussyhunters', name: 'TS Pussy Hunters', url: 'https://www.kink.com/channel/tspussyhunters', description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.', diff --git a/src/fetch-releases.js b/src/fetch-releases.js index 7a901b12..53ce2140 100644 --- a/src/fetch-releases.js +++ b/src/fetch-releases.js @@ -1,8 +1,11 @@ 'use strict'; const config = require('config'); +const fs = require('fs-extra'); +const path = require('path'); const Promise = require('bluebird'); const moment = require('moment'); +const bhttp = require('bhttp'); const argv = require('./argv'); const knex = require('./knex'); @@ -75,9 +78,7 @@ async function findDuplicateReleases(latestReleases, _siteId) { } async function storeReleases(releases = []) { - return Promise.reduce(releases, async (acc, release) => { - await acc; - + return Promise.map(releases, async (release) => { const curatedRelease = { site_id: release.site.id, shoot_id: release.shootId || null, @@ -115,7 +116,23 @@ async function storeReleases(releases = []) { release_id: releaseEntry.rows[0].id, }))); } - }, []); + + if (release.thumbnails && release.thumbnails.length > 0) { + const thumbnailPath = path.join(config.thumbnailPath, release.site.id, releaseEntry.rows[0].id.toString()); + + await fs.mkdir(thumbnailPath, { recursive: true }); + + await Promise.map(release.thumbnails, async (thumbnailUrl, index) => { + const res = await bhttp.get(thumbnailUrl); + + await fs.writeFile(path.join(thumbnailPath, `${index}.jpg`), res.body); + }, { + concurrency: 2, + }); + } + }, { + concurrency: 2, + }); } async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) { @@ -151,7 +168,7 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page async function fetchReleases() { const sites = await accumulateIncludedSites(); - const scenesPerSite = await Promise.all(sites.map(async (site) => { + const scenesPerSite = await Promise.map(sites, async (site) => { const scraper = scrapers[site.id] || scrapers[site.network.id]; if (scraper) { @@ -167,13 +184,18 @@ async function fetchReleases() { if (argv.save) { const finalReleases = argv.deep - ? await Promise.all(newReleases.map(async (release) => { + ? await Promise.map(newReleases, async (release) => { if (release.url) { - return fetchScene(release.url); + const scene = await fetchScene(release.url, release); + + return { + ...release, + ...scene, + }; } return release; - }), { + }, { concurrency: 2, }) : newReleases; @@ -204,7 +226,9 @@ async function fetchReleases() { } return []; - })); + }, { + concurrency: 2, + }); const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []); const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA)); diff --git a/src/releases.js b/src/releases.js index 490d33b0..ea4defe1 100644 --- a/src/releases.js +++ b/src/releases.js @@ -32,7 +32,11 @@ async function curateRelease(release) { site: { id: release.site_id, name: release.site_name, - network: release.network_id, + }, + network: { + id: release.network_id, + name: release.network_name, + url: release.network_url, }, }; } @@ -43,8 +47,9 @@ function curateReleases(releases) { async function fetchReleases() { const releases = await knex('releases') - .select('releases.*', 'sites.name as site_name') + .select('releases.*', 'sites.name as site_name', 'sites.network_id', 'networks.name as network_name', 'networks.url as network_url') .leftJoin('sites', 'releases.site_id', 'sites.id') + .leftJoin('networks', 'sites.network_id', 'networks.id') .orderBy('date', 'desc') .limit(100); diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index f142bdb6..5f484491 100644 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -11,6 +11,10 @@ function scrapeLatest(html, site) { const scenesElements = $('.update_details').toArray(); return scenesElements.map((element) => { + const thumbnailElement = $(element).find('a img.thumbs'); + const thumbnailCount = Number(thumbnailElement.attr('cnt')); + const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined); + const sceneLinkElement = $(element).children('a').eq(1); const url = sceneLinkElement.attr('href'); const title = sceneLinkElement.text(); @@ -32,6 +36,7 @@ function scrapeLatest(html, site) { actors, date, site, + thumbnails, }; }); } @@ -41,6 +46,10 @@ function scrapeUpcoming(html, site) { const scenesElements = $('#coming_soon_carousel').find('.table').toArray(); return scenesElements.map((element) => { + const thumbnailElement = $(element).find('a img.thumbs'); + const thumbnailCount = Number(thumbnailElement.attr('cnt')); + const thumbnails = Array.from({ length: thumbnailCount }, (value, index) => thumbnailElement.attr(`src${index}_1x`)).filter(thumbnailUrl => thumbnailUrl !== undefined); + const shootId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0]; const details = $(element).find('.update_details_comingsoon') @@ -66,8 +75,9 @@ function scrapeUpcoming(html, site) { url: null, shootId, title, - actors, date, + actors, + thumbnails, rating: null, site, }; diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js index ef2e56c8..24fbed0a 100644 --- a/src/scrapers/kink.js +++ b/src/scrapers/kink.js @@ -18,6 +18,8 @@ function scrapeLatest(html, site) { const shootId = href.split('/')[2]; const title = sceneLinkElement.text().trim(); + const thumbnails = $(element).find('.rollover .roll-image').map((thumbnailIndex, thumbnailElement) => $(thumbnailElement).attr('data-imagesrc')).toArray(); + const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate(); const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const stars = $(element).find('.average-rating').attr('data-rating') / 10; @@ -33,6 +35,7 @@ function scrapeLatest(html, site) { title, actors, date, + thumbnails, rating: { stars, }, @@ -49,6 +52,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) { const title = $('h1.shoot-title span.favorite-button').attr('data-title'); const actorsRaw = $('.shoot-info p.starring'); + const thumbnails = $('.gallery .thumb img').map((thumbnailIndex, thumbnailElement) => `https://cdnp.kink.com${$(thumbnailElement).attr('data-image-file')}`).toArray(); + const trailerVideo = $('.player span[data-type="trailer-src"]').attr('data-url'); + const trailerPoster = $('.player video#kink-player').attr('poster'); + const date = moment.utc($(actorsRaw) .prev() .text() @@ -78,6 +85,15 @@ async function scrapeScene(html, url, shootId, ratingRes, site) { date, actors, description, + thumbnails, + trailer: { + video: { + default: trailerVideo, + sd: trailerVideo, + hd: trailerVideo.replace('480p', '720p'), + }, + poster: trailerPoster, + }, rating: { stars, }, diff --git a/src/web/server.js b/src/web/server.js index a86b91de..c7145dec 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -12,6 +12,9 @@ function initServer() { const app = express(); const router = Router(); + app.use(express.static(config.thumbnailPath)); + app.use(express.static('public')); + app.set('views', path.join(__dirname, '../../assets/views')); app.set('view engine', 'jsx'); app.engine('jsx', createEngine());