Refactored 21sextury scraper.

This commit is contained in:
ThePendulum 2019-12-09 05:00:49 +01:00
parent d874c508de
commit 04a89efa58
52 changed files with 2621 additions and 2068 deletions

View File

@ -3,5 +3,6 @@
[ [
"@babel/preset-env" "@babel/preset-env"
] ]
] ],
"plugins": ["@babel/plugin-proposal-optional-chaining"]
} }

View File

@ -2,6 +2,7 @@
"root": true, "root": true,
"extends": ["airbnb-base", "plugin:vue/recommended"], "extends": ["airbnb-base", "plugin:vue/recommended"],
"parserOptions": { "parserOptions": {
"parser": "babel-eslint",
"ecmaVersion": 2019, "ecmaVersion": 2019,
"sourceType": "module" "sourceType": "module"
}, },

View File

@ -293,6 +293,7 @@ export default {
.logo { .logo {
display: inline-block; display: inline-block;
filter: drop-shadow(0 0 1px $shadow);
} }
.logo-site { .logo-site {
@ -300,6 +301,7 @@ export default {
max-width: 15rem; max-width: 15rem;
object-fit: contain; object-fit: contain;
object-position: 100% 50%; object-position: 100% 50%;
filter: drop-shadow(0 0 1px $shadow);
} }
.logo-network { .logo-network {

View File

@ -9,7 +9,7 @@
<div class="sidebar"> <div class="sidebar">
<a <a
v-if="tag.poster" v-if="tag.poster"
:href="`/media/${tag.poster.path}`" :href="`/img/${tag.poster.path}`"
:title="tag.poster.comment" :title="tag.poster.comment"
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"

View File

@ -98,6 +98,8 @@ export default {
</script> </script>
<style lang="scss" scoped> <style lang="scss" scoped>
@import 'theme';
.tags { .tags {
padding: 1rem; padding: 1rem;
} }

View File

@ -30,8 +30,8 @@ export default {
@import 'theme'; @import 'theme';
.tile { .tile {
color: $text-contrast; color: $text;
background: $profile; background: $background;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
align-items: center; align-items: center;

View File

@ -86,5 +86,6 @@ module.exports = {
media: { media: {
path: './', path: './',
thumbnailSize: 320, // width for 16:9 will be exactly 576px thumbnailSize: 320, // width for 16:9 will be exactly 576px
limit: 25, // max number of photos per release
}, },
}; };

4002
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -5,10 +5,12 @@
"main": "src/app.js", "main": "src/app.js",
"scripts": { "scripts": {
"postinstall": "npm run migrate && npm run seed", "postinstall": "npm run migrate && npm run seed",
"start": "node src/app.js", "start": "node dist/init.js",
"webpack": "webpack --env=production --mode=production", "webpack": "webpack --env=production --mode=production",
"webpack-dev": "webpack --env=development --mode=development", "webpack-dev": "webpack --env=development --mode=development",
"webpack-watch": "webpack --progress --colors --watch --env=development --mode=development", "webpack-watch": "webpack --progress --colors --watch --env=development --mode=development",
"babel": "babel src -d dist",
"babel-watch": "babel src -w -d dist",
"eslint": "eslint src/", "eslint": "eslint src/",
"eslint-watch": "esw --watch src/", "eslint-watch": "esw --watch src/",
"knex": "knex", "knex": "knex",
@ -32,68 +34,70 @@
"author": "Niels Simenon", "author": "Niels Simenon",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@babel/cli": "^7.2.3", "@babel/cli": "^7.7.5",
"@babel/core": "^7.4.5", "@babel/core": "^7.7.5",
"@babel/preset-env": "^7.4.5", "@babel/plugin-proposal-optional-chaining": "^7.7.5",
"autoprefixer": "^9.5.1", "@babel/preset-env": "^7.7.6",
"autoprefixer": "^9.7.3",
"babel-cli": "^6.26.0", "babel-cli": "^6.26.0",
"babel-eslint": "^10.0.1", "babel-eslint": "^10.0.3",
"babel-loader": "^8.0.6", "babel-loader": "^8.0.6",
"babel-preset-airbnb": "^3.2.0", "babel-preset-airbnb": "^3.3.2",
"babel-register": "^6.26.0", "babel-register": "^6.26.0",
"css-loader": "^2.1.1", "css-loader": "^2.1.1",
"eslint": "^5.16.0", "eslint": "^5.16.0",
"eslint-config-airbnb": "^17.1.0", "eslint-config-airbnb": "^17.1.1",
"eslint-config-airbnb-base": "^13.1.0", "eslint-config-airbnb-base": "^13.2.0",
"eslint-loader": "^2.1.2", "eslint-loader": "^2.2.1",
"eslint-plugin-import": "^2.17.3", "eslint-plugin-import": "^2.18.2",
"eslint-plugin-jsx-a11y": "^6.2.1", "eslint-plugin-jsx-a11y": "^6.2.3",
"eslint-plugin-react": "^7.13.0", "eslint-plugin-react": "^7.17.0",
"eslint-plugin-vue": "^5.2.2", "eslint-plugin-vue": "^6.0.1",
"eslint-watch": "^4.0.2", "eslint-watch": "^4.0.2",
"mini-css-extract-plugin": "^0.7.0", "mini-css-extract-plugin": "^0.7.0",
"node-sass": "^4.12.0", "node-sass": "^4.13.0",
"postcss-loader": "^3.0.0", "postcss-loader": "^3.0.0",
"raw-loader": "^2.0.0", "raw-loader": "^2.0.0",
"sass-loader": "^7.1.0", "sass-loader": "^7.3.1",
"style-loader": "^0.23.1", "style-loader": "^0.23.1",
"vue-loader": "^15.7.0", "vue-loader": "^15.7.2",
"vue-template-compiler": "^2.6.10", "vue-template-compiler": "^2.6.10",
"webpack": "^4.32.2", "webpack": "^4.41.2",
"webpack-cli": "^3.3.2" "webpack-cli": "^3.3.10"
}, },
"dependencies": { "dependencies": {
"babel-polyfill": "^6.26.0",
"bhttp": "^1.2.4", "bhttp": "^1.2.4",
"blake2": "^4.0.0", "blake2": "^4.0.0",
"bluebird": "^3.5.4", "bluebird": "^3.7.2",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"cheerio": "^1.0.0-rc.2", "cheerio": "^1.0.0-rc.3",
"cli-confirm": "^1.0.1", "cli-confirm": "^1.0.1",
"config": "^3.0.1", "config": "^3.2.4",
"dayjs": "^1.8.14", "dayjs": "^1.8.17",
"express": "^4.16.4", "express": "^4.17.1",
"express-promise-router": "^3.0.3", "express-promise-router": "^3.0.3",
"express-react-views": "^0.11.0", "express-react-views": "^0.11.0",
"fs-extra": "^7.0.1", "fs-extra": "^7.0.1",
"jsdom": "^15.2.0", "jsdom": "^15.2.1",
"knex": "^0.16.3", "knex": "^0.16.5",
"knex-migrate": "^1.7.1", "knex-migrate": "^1.7.4",
"mime": "^2.4.4", "mime": "^2.4.4",
"moment": "^2.24.0", "moment": "^2.24.0",
"opn": "^5.4.0", "opn": "^5.5.0",
"pg": "^7.9.0", "pg": "^7.14.0",
"prop-types": "^15.7.2", "prop-types": "^15.7.2",
"react": "^16.8.6", "react": "^16.12.0",
"react-dom": "^16.8.6", "react-dom": "^16.12.0",
"sharp": "^0.23.2", "sharp": "^0.23.4",
"showdown": "^1.9.1", "showdown": "^1.9.1",
"tough-cookie": "^3.0.1", "tough-cookie": "^3.0.1",
"tty-table": "^2.7.0", "tty-table": "^2.8.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.0.2", "v-tooltip": "^2.0.2",
"vue": "^2.6.10", "vue": "^2.6.10",
"vue-router": "^3.0.6", "vue-router": "^3.1.3",
"vuex": "^3.1.1", "vuex": "^3.1.2",
"yargs": "^13.2.2" "yargs": "^13.3.0"
} }
} }

View File

@ -42,7 +42,9 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.filter-bar[data-v-6db17c96] { .filter-bar[data-v-6db17c96] {
background: #fff; background: #fff;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
padding: .5rem 1rem; padding: .5rem 1rem;
z-index: 1; z-index: 1;
@ -93,7 +95,10 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.tile[data-v-3abcf101] { .tile[data-v-3abcf101] {
background: #fff; background: #fff;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
box-sizing: border-box; box-sizing: border-box;
padding: 0 0 .5rem 0; padding: 0 0 .5rem 0;
@ -108,8 +113,11 @@
.thumbnail[data-v-3abcf101] { .thumbnail[data-v-3abcf101] {
width: 100%; width: 100%;
height: 12rem; height: 12rem;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
-webkit-box-align: center;
align-items: center; align-items: center;
-o-object-fit: cover; -o-object-fit: cover;
object-fit: cover; object-fit: cover;
@ -120,8 +128,11 @@
text-shadow: 1px 1px 0 rgba(255, 255, 255, 0.5); text-shadow: 1px 1px 0 rgba(255, 255, 255, 0.5);
} }
.row[data-v-3abcf101] { .row[data-v-3abcf101] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
-webkit-box-align: center;
align-items: center; align-items: center;
box-sizing: border-box; box-sizing: border-box;
padding: 0 .5rem; padding: 0 .5rem;
@ -129,8 +140,11 @@
} }
.details[data-v-3abcf101] { .details[data-v-3abcf101] {
width: 100%; width: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
position: absolute; position: absolute;
font-size: 0; font-size: 0;
@ -162,8 +176,12 @@
font-weight: bold; font-weight: bold;
} }
.info[data-v-3abcf101] { .info[data-v-3abcf101] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
} }
.link[data-v-3abcf101] { .link[data-v-3abcf101] {
@ -281,8 +299,11 @@
background: rgba(0, 0, 0, 0.1); background: rgba(0, 0, 0, 0.1);
height: 12rem; height: 12rem;
width: 100%; width: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
-o-object-fit: cover; -o-object-fit: cover;
object-fit: cover; object-fit: cover;
@ -324,7 +345,9 @@
box-sizing: border-box; box-sizing: border-box;
} }
.row[data-v-2bc41e74] { .row[data-v-2bc41e74] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
margin: 0 0 1rem 0; margin: 0 0 1rem 0;
} }
@ -341,7 +364,9 @@
cursor: default; cursor: default;
} }
.info .column[data-v-2bc41e74] { .info .column[data-v-2bc41e74] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
padding: 0 1rem; padding: 0 1rem;
} }
@ -362,15 +387,21 @@
margin: 0 1rem 0 0; margin: 0 1rem 0 0;
} }
.site[data-v-2bc41e74] { .site[data-v-2bc41e74] {
display: -webkit-inline-box;
display: inline-flex; display: inline-flex;
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: end;
justify-content: flex-end; justify-content: flex-end;
padding: .25rem 0; padding: .25rem 0;
font-size: 0; font-size: 0;
} }
.logo[data-v-2bc41e74] { .logo[data-v-2bc41e74] {
display: inline-block; display: inline-block;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
} }
.logo-site[data-v-2bc41e74] { .logo-site[data-v-2bc41e74] {
height: 3rem; height: 3rem;
@ -379,6 +410,8 @@
object-fit: contain; object-fit: contain;
-o-object-position: 100% 50%; -o-object-position: 100% 50%;
object-position: 100% 50%; object-position: 100% 50%;
-webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
} }
.logo-network[data-v-2bc41e74] { .logo-network[data-v-2bc41e74] {
height: 1.5rem; height: 1.5rem;
@ -407,6 +440,7 @@
font-size: 1rem; font-size: 1rem;
} }
.actors[data-v-2bc41e74] { .actors[data-v-2bc41e74] {
display: -webkit-box;
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
} }
@ -460,12 +494,16 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.header[data-v-3e57cf44] { .header[data-v-3e57cf44] {
display: -webkit-box;
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
} }
.title[data-v-3e57cf44] { .title[data-v-3e57cf44] {
display: -webkit-inline-box;
display: inline-flex; display: inline-flex;
-webkit-box-align: top;
align-items: top; align-items: top;
margin: 0 1rem 0 0; margin: 0 1rem 0 0;
} }
@ -477,9 +515,13 @@
margin: 0 0 1rem 0; margin: 0 0 1rem 0;
} }
.link[data-v-3e57cf44] { .link[data-v-3e57cf44] {
display: -webkit-box;
display: flex; display: flex;
flex-shrink: 0; flex-shrink: 0;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-align: end;
align-items: flex-end; align-items: flex-end;
} }
.logo[data-v-3e57cf44] { .logo[data-v-3e57cf44] {
@ -490,7 +532,9 @@
margin: 0 .5rem 1rem 0; margin: 0 .5rem 1rem 0;
} }
.networklogo-container[data-v-3e57cf44] { .networklogo-container[data-v-3e57cf44] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
} }
.networklogo[data-v-3e57cf44] { .networklogo[data-v-3e57cf44] {
@ -517,8 +561,12 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.tile[data-v-f4958086] { .tile[data-v-f4958086] {
background: #fff; background: #fff;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-align: center;
align-items: center; align-items: center;
box-sizing: border-box; box-sizing: border-box;
padding: .5rem 1rem; padding: .5rem 1rem;
@ -534,8 +582,11 @@
width: 100%; width: 100%;
height: 5rem; height: 5rem;
color: #222; color: #222;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
-o-object-fit: contain; -o-object-fit: contain;
object-fit: contain; object-fit: contain;
@ -547,21 +598,28 @@
.title[data-v-f4958086] { .title[data-v-f4958086] {
color: #222; color: #222;
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
margin: 0; margin: 0;
} }
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.header[data-v-e2e12602] { .header[data-v-e2e12602] {
display: -webkit-box;
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
-webkit-box-align: top;
align-items: top; align-items: top;
margin: 0 0 2rem 0; margin: 0 0 2rem 0;
} }
.title[data-v-e2e12602] { .title[data-v-e2e12602] {
display: -webkit-inline-box;
display: inline-flex; display: inline-flex;
-webkit-box-align: top;
align-items: top; align-items: top;
margin: 0 1rem 0 0; margin: 0 1rem 0 0;
} }
@ -592,8 +650,12 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.tile[data-v-8b4c90b0] { .tile[data-v-8b4c90b0] {
background: #fff; background: #fff;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-align: center;
align-items: center; align-items: center;
box-sizing: border-box; box-sizing: border-box;
padding: .5rem 1rem; padding: .5rem 1rem;
@ -609,8 +671,11 @@
width: 100%; width: 100%;
height: 5rem; height: 5rem;
color: #222; color: #222;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
-o-object-fit: contain; -o-object-fit: contain;
object-fit: contain; object-fit: contain;
@ -622,7 +687,9 @@
.title[data-v-8b4c90b0] { .title[data-v-8b4c90b0] {
color: #222; color: #222;
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
margin: 0; margin: 0;
} }
@ -660,6 +727,7 @@
.photos[data-v-0a0430c7] { .photos[data-v-0a0430c7] {
width: 100%; width: 100%;
max-width: 100%; max-width: 100%;
display: -webkit-box;
display: flex; display: flex;
overflow-x: scroll; overflow-x: scroll;
scrollbar-width: none; scrollbar-width: none;
@ -680,7 +748,10 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.actor-inner[data-v-ea0483c2] { .actor-inner[data-v-ea0483c2] {
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
padding: 0; padding: 0;
overflow-x: auto; overflow-x: auto;
@ -689,7 +760,10 @@
background: #222; background: #222;
color: rgba(255, 255, 255, 0.9); color: rgba(255, 255, 255, 0.9);
width: 100%; width: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: horizontal;
-webkit-box-direction: normal;
flex-direction: row; flex-direction: row;
flex-shrink: 0; flex-shrink: 0;
} }
@ -708,6 +782,7 @@
object-position: 50% 0; object-position: 50% 0;
} }
.bio[data-v-ea0483c2] { .bio[data-v-ea0483c2] {
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
min-width: 20rem; min-width: 20rem;
box-sizing: border-box; box-sizing: border-box;
@ -715,13 +790,18 @@
margin: 0 2rem 0 0; margin: 0 2rem 0 0;
} }
.bio-header[data-v-ea0483c2] { .bio-header[data-v-ea0483c2] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
-webkit-box-align: center;
align-items: center; align-items: center;
margin: 0 0 1rem 0; margin: 0 0 1rem 0;
} }
.bio-item[data-v-ea0483c2] { .bio-item[data-v-ea0483c2] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-pack: justify;
justify-content: space-between; justify-content: space-between;
padding: 0 0 .25rem 0; padding: 0 0 .25rem 0;
margin: 0 0 .25rem 0; margin: 0 0 .25rem 0;
@ -801,6 +881,7 @@
font-size: .8rem; font-size: .8rem;
} }
.extra[data-v-ea0483c2] { .extra[data-v-ea0483c2] {
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
} }
.description[data-v-ea0483c2] { .description[data-v-ea0483c2] {
@ -837,8 +918,12 @@
fill: #ff6c88; fill: #ff6c88;
} }
.actor-content[data-v-ea0483c2] { .actor-content[data-v-ea0483c2] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
-webkit-box-orient: horizontal;
-webkit-box-direction: normal;
flex-direction: row; flex-direction: row;
} }
.heading[data-v-ea0483c2] { .heading[data-v-ea0483c2] {
@ -856,6 +941,7 @@
display: none; display: none;
} }
.releases[data-v-ea0483c2] { .releases[data-v-ea0483c2] {
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
padding: 1rem; padding: 1rem;
} }
@ -870,6 +956,8 @@
display: none; display: none;
} }
.actor-content[data-v-ea0483c2] { .actor-content[data-v-ea0483c2] {
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
} }
.photos-container[data-v-ea0483c2] { .photos-container[data-v-ea0483c2] {
@ -882,11 +970,14 @@
display: none; display: none;
} }
.photos.compact[data-v-ea0483c2] { .photos.compact[data-v-ea0483c2] {
display: -webkit-box;
display: flex; display: flex;
} }
} }
@media (max-width: 720px) { @media (max-width: 720px) {
.profile[data-v-ea0483c2] { .profile[data-v-ea0483c2] {
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
padding: 0 0 .5rem 0; padding: 0 0 .5rem 0;
} }
@ -931,9 +1022,14 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.tag[data-v-7f130e7f] { .tag[data-v-7f130e7f] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: horizontal;
-webkit-box-direction: normal;
flex-direction: row; flex-direction: row;
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
-webkit-box-pack: stretch;
justify-content: stretch; justify-content: stretch;
} }
.sidebar[data-v-7f130e7f] { .sidebar[data-v-7f130e7f] {
@ -970,10 +1066,14 @@
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.tile[data-v-602c6fd8] { .tile[data-v-602c6fd8] {
color: #fff; color: #222;
background: #222; background: #fff;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-align: center;
align-items: center; align-items: center;
box-sizing: border-box; box-sizing: border-box;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.25); box-shadow: 0 0 3px rgba(0, 0, 0, 0.25);
@ -988,8 +1088,11 @@
} }
.title[data-v-602c6fd8] { .title[data-v-602c6fd8] {
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
font-size: 1rem; font-size: 1rem;
padding: .5rem 1rem; padding: .5rem 1rem;
@ -997,6 +1100,7 @@
text-transform: capitalize; text-transform: capitalize;
} }
/* $primary: #ff886c; */
.tags[data-v-66fa6284] { .tags[data-v-66fa6284] {
padding: 1rem; padding: 1rem;
} }
@ -1011,9 +1115,14 @@
background: #fff; background: #fff;
color: #ff6c88; color: #ff6c88;
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
font-size: 2rem; font-size: 2rem;
} }
@ -1149,7 +1258,9 @@ body {
/* $primary: #ff886c; */ /* $primary: #ff886c; */
.header[data-v-10b7ec04] { .header[data-v-10b7ec04] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
background: #fff; background: #fff;
color: #ff6c88; color: #ff6c88;
@ -1171,8 +1282,11 @@ body {
display: inline-block; display: inline-block;
} }
.nav-link[data-v-10b7ec04] { .nav-link[data-v-10b7ec04] {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-align: center;
align-items: center; align-items: center;
-webkit-box-pack: center;
justify-content: center; justify-content: center;
padding: 1rem; padding: 1rem;
border-bottom: solid 5px transparent; border-bottom: solid 5px transparent;
@ -1203,10 +1317,12 @@ body {
display: none; display: none;
} }
.nav .nolist[data-v-10b7ec04] { .nav .nolist[data-v-10b7ec04] {
display: -webkit-box;
display: flex; display: flex;
} }
.nav[data-v-10b7ec04], .nav[data-v-10b7ec04],
.nav-item[data-v-10b7ec04] { .nav-item[data-v-10b7ec04] {
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
} }
} }
@ -1215,17 +1331,25 @@ body {
.container { .container {
background: #fafafa; background: #fafafa;
height: 100%; height: 100%;
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
overflow: hidden; overflow: hidden;
} }
.content { .content {
display: -webkit-box;
display: flex; display: flex;
-webkit-box-orient: vertical;
-webkit-box-direction: normal;
flex-direction: column; flex-direction: column;
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
overflow-y: auto; overflow-y: auto;
} }
.content-inner { .content-inner {
-webkit-box-flex: 1;
flex-grow: 1; flex-grow: 1;
padding: 1rem; padding: 1rem;
overflow-y: auto; overflow-y: auto;

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.0 KiB

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

BIN
public/img/tags/tattoo.jpeg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 106 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

View File

@ -21,23 +21,17 @@ function getSites(networksMap) {
{ {
slug: 'buttplays', slug: 'buttplays',
name: 'Butt Plays', name: 'Butt Plays',
url: 'https://www.buttplays.com',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
parameters: JSON.stringify({ filter: true }),
}, },
{ {
slug: 'clubsandy', slug: 'clubsandy',
name: 'Club Sandy', name: 'Club Sandy',
url: 'https://www.clubsandy.com',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
parameters: JSON.stringify({ filter: true }),
}, },
{ {
slug: 'deepthroatfrenzy', slug: 'deepthroatfrenzy',
name: 'Deepthroat Frenzy', name: 'Deepthroat Frenzy',
url: 'https://www.deepthroatfrenzy.com',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
parameters: JSON.stringify({ filter: true }),
}, },
{ {
slug: 'dpfanatics', slug: 'dpfanatics',
@ -56,9 +50,7 @@ function getSites(networksMap) {
{ {
slug: 'gapeland', slug: 'gapeland',
name: 'Gapeland', name: 'Gapeland',
url: 'https://www.gapeland.com',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
parameters: JSON.stringify({ filter: true }),
}, },
{ {
slug: 'lezcuties', slug: 'lezcuties',
@ -67,12 +59,65 @@ function getSites(networksMap) {
description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.', description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
}, },
{
slug: 'onlyswallows',
name: 'Only Swallows',
network_id: networksMap['21sextury'],
},
{
slug: 'alettaoceanempire',
name: 'Aletta Ocean Empire',
network_id: networksMap['21sextury'],
},
{
slug: 'analqueenalysa',
name: 'Anal Queen Alysa',
network_id: networksMap['21sextury'],
},
{
slug: 'blueangellive',
name: 'Blue Angel Live',
network_id: networksMap['21sextury'],
},
{ {
slug: 'pixandvideo', slug: 'pixandvideo',
name: 'Pix and Video', name: 'Pix and Video',
url: 'https://www.pixandvideo.com',
network_id: networksMap['21sextury'], network_id: networksMap['21sextury'],
parameters: JSON.stringify({ filter: true }), },
{
slug: 'cheatingwhorewives',
name: 'Cheating Whore Wives',
network_id: networksMap['21sextury'],
},
{
slug: 'cutiesgalore',
name: 'Cuties Galore',
network_id: networksMap['21sextury'],
},
{
slug: 'hotmilfclub',
name: 'Hot MILF Club',
network_id: networksMap['21sextury'],
},
{
slug: 'letsplaylez',
name: 'Lets Play Lez',
network_id: networksMap['21sextury'],
},
{
slug: 'nudefightclub',
name: 'Nude Fight Club',
network_id: networksMap['21sextury'],
},
{
slug: 'sexwithkathianobili',
name: 'Sex With Kathia Nobili',
network_id: networksMap['21sextury'],
},
{
slug: 'sweetsophiemoone',
name: 'Sweet Sophie Moone',
network_id: networksMap['21sextury'],
}, },
// BANGBROS // BANGBROS
{ {

View File

@ -72,6 +72,12 @@ function getMedia(tagsMap) {
role: 'poster', role: 'poster',
comment: 'Juelz Ventura in "Gangbanged 5" for Elegant Angel', comment: 'Juelz Ventura in "Gangbanged 5" for Elegant Angel',
}, },
{
path: 'tags/tattoo.jpeg',
target_id: tagsMap.tattoo,
role: 'poster',
comment: 'Kali Roses in "Goes All In For Anal" for Hussie Pass',
},
{ {
path: 'tags/triple-anal.jpeg', path: 'tags/triple-anal.jpeg',
target_id: tagsMap['triple-anal'], target_id: tagsMap['triple-anal'],

View File

@ -1,13 +1,15 @@
{ {
"extends": "airbnb-base", "extends": "airbnb-base",
"parserOptions": { "parserOptions": {
"parser": "babel-eslint",
"sourceType": "script" "sourceType": "script"
}, },
"rules": { "rules": {
"strict": 0, "strict": 0,
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0, "no-console": 0,
"indent": ["error", 4], "indent": "off",
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}] "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
"template-curly-spacing": "off"
} }
} }

View File

@ -271,6 +271,8 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
} }
async function mergeProfiles(profiles, actor) { async function mergeProfiles(profiles, actor) {
console.log(profiles);
const mergedProfile = profiles.reduce((prevProfile, profile) => { const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) { if (profile === null) {
return prevProfile; return prevProfile;

View File

@ -46,4 +46,4 @@ async function init() {
await initServer(); await initServer();
} }
init(); module.exports = init;

View File

@ -63,6 +63,7 @@ const { argv } = yargs
.option('debug', { .option('debug', {
describe: 'Show error stack traces', describe: 'Show error stack traces',
type: 'boolean', type: 'boolean',
default: process.env.NODE_ENV === 'development',
}); });
module.exports = argv; module.exports = argv;

4
src/init.js Normal file
View File

@ -0,0 +1,4 @@
require('babel-polyfill');
const init = require('./app');
init();

View File

@ -10,6 +10,7 @@ const sharp = require('sharp');
const blake2 = require('blake2'); const blake2 = require('blake2');
const knex = require('./knex'); const knex = require('./knex');
const pluckPhotos = require('./utils/pluck-photos');
function getHash(buffer) { function getHash(buffer) {
const hash = blake2.createHash('blake2b', { digestLength: 24 }); const hash = blake2.createHash('blake2b', { digestLength: 24 });
@ -94,10 +95,10 @@ async function filterHashDuplicates(files, domains = ['releases'], roles = ['pho
} }
async function fetchPhoto(photoUrl, index, identifier) { async function fetchPhoto(photoUrl, index, identifier) {
try {
const { pathname } = new URL(photoUrl); const { pathname } = new URL(photoUrl);
const mimetype = mime.getType(pathname); const mimetype = mime.getType(pathname);
try {
const res = await bhttp.get(photoUrl); const res = await bhttp.get(photoUrl);
if (res.statusCode === 200) { if (res.statusCode === 200) {
@ -176,7 +177,11 @@ async function storePhotos(release, releaseId) {
return; return;
} }
const newPhotos = await filterSourceDuplicates(release.photos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`); const pluckedPhotos = pluckPhotos(release.photos, release);
console.log(release.photos, pluckedPhotos);
const newPhotos = await filterSourceDuplicates(pluckedPhotos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
if (newPhotos.length === 0) return; if (newPhotos.length === 0) return;

View File

@ -125,7 +125,7 @@ async function curateScrapedRelease(release) {
likes: release.rating && release.rating.likes, likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes, dislikes: release.rating && release.rating.dislikes,
rating: release.rating && release.rating.stars && Math.floor(release.rating.stars), rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
deep: Boolean(argv.deep && release.url && !release.upcoming), deep: typeof release.deep === 'boolean' ? release.deep : false,
}; };
if (release.site.isFallback && release.channel) { if (release.site.isFallback && release.channel) {
@ -275,6 +275,12 @@ async function storeRelease(release) {
async function storeReleases(releases) { async function storeReleases(releases) {
const storedReleases = await Promise.map(releases, async (release) => { const storedReleases = await Promise.map(releases, async (release) => {
if (release.site.isFallback && !release.channel) {
console.error(`Unable to derive channel site from generic URL: ${release.url}.`);
return null;
}
try { try {
const releaseId = await storeRelease(release); const releaseId = await storeRelease(release);
@ -289,7 +295,7 @@ async function storeReleases(releases) {
} }
}, { }, {
concurrency: 10, concurrency: 10,
}); }).filter(release => release);
const actors = storedReleases.reduce((acc, release) => { const actors = storedReleases.reduce((acc, release) => {
if (!release.actors) return acc; if (!release.actors) return acc;

View File

@ -51,8 +51,10 @@ async function scrapeRelease(url, release, deep = false) {
// don't store release when called by site scraper // don't store release when called by site scraper
const [storedRelease] = await storeReleases([scene]); const [storedRelease] = await storeReleases([scene]);
if (storedRelease) {
console.log(`http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`); console.log(`http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`);
} }
}
return scene; return scene;
} }

View File

@ -69,12 +69,20 @@ async function scrapeUpcomingReleases(scraper, site) {
async function deepFetchReleases(baseReleases) { async function deepFetchReleases(baseReleases) {
return Promise.map(baseReleases, async (release) => { return Promise.map(baseReleases, async (release) => {
if (release.url) { if (release.url) {
try {
const fullRelease = await scrapeRelease(release.url, release, true); const fullRelease = await scrapeRelease(release.url, release, true);
return { return {
...release, ...release,
...fullRelease, ...fullRelease,
deep: true,
}; };
} catch (error) {
return {
...release,
deep: false,
};
}
} }
return release; return release;
@ -116,7 +124,7 @@ async function scrapeReleases() {
return await scrapeSiteReleases(scraper, site); return await scrapeSiteReleases(scraper, site);
} catch (error) { } catch (error) {
if (argv.debug) { if (argv.debug) {
console.error(`${site.id}: Failed to scrape releases`, error); console.error(`${site.name}: Failed to scrape releases`, error);
} }
console.warn(`${site.id}: Failed to scrape releases`); console.warn(`${site.id}: Failed to scrape releases`);

View File

@ -1,11 +1,51 @@
'use strict'; 'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const moment = require('moment'); const moment = require('moment');
const knex = require('../knex'); async function fetchPhotos(photoPath) {
const { matchTags } = require('../tags'); const res = await bhttp.get(`https://21sextury.com${photoPath}`);
return res.body.toString();
}
function scrapePhotos(html) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
.map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
.map((photoIndex, photoElement) => $(photoElement)
.attr('src'))
// .replace('_tb.jpg', '.jpg')) does not always work
.toArray();
return unlockedPhotos.concat(lockedThumbnails);
}
async function getPhotos(photoPath) {
if (!photoPath || photoPath.match('join')) {
return [];
}
const html = await fetchPhotos(photoPath);
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = scrapePhotos(html);
const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
const otherPhotos = await Promise.map(pages, async (pagePath) => {
const pageHtml = await fetchPhotos(pagePath);
return scrapePhotos(pageHtml);
}, {
concurrency: 2,
});
return photos.concat(otherPhotos.flat());
}
function scrape(html, site) { function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
@ -14,12 +54,13 @@ function scrape(html, site) {
return scenesElements.reduce((accReleases, element) => { return scenesElements.reduce((accReleases, element) => {
const siteName = $(element).find('.studioName a').attr('title'); const siteName = $(element).find('.studioName a').attr('title');
if (site.parameters && site.parameters.filter && siteName.toLowerCase() !== site.name.toLowerCase()) { if (!site.url && siteName.toLowerCase() !== site.name.toLowerCase()) {
// using generic overview as fallback, scene from different site
return accReleases; return accReleases;
} }
const sceneLinkElement = $(element).find('.sceneTitle a'); const sceneLinkElement = $(element).find('.sceneTitle a');
const url = `${site.url}${sceneLinkElement.attr('href')}`; const url = `${site.url || 'https://www.21sextury.com'}${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title').trim(); const title = sceneLinkElement.attr('title').trim();
const entryId = $(element).attr('data-itemid'); const entryId = $(element).attr('data-itemid');
@ -32,6 +73,9 @@ function scrape(html, site) {
.map((actorIndex, actorElement) => $(actorElement).attr('title')) .map((actorIndex, actorElement) => $(actorElement).attr('title'))
.toArray(); .toArray();
const poster = $(element).find('.imgLink img').attr('data-original');
const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
const [likes, dislikes] = $(element).find('.value') const [likes, dislikes] = $(element).find('.value')
.toArray() .toArray()
.map(value => Number($(value).text())); .map(value => Number($(value).text()));
@ -44,6 +88,10 @@ function scrape(html, site) {
title, title,
actors, actors,
date, date,
poster,
trailer: {
src: trailer,
},
rating: { rating: {
likes, likes,
dislikes, dislikes,
@ -58,25 +106,21 @@ async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElement = $('#videoWrapper'); const sceneElement = $('#videoWrapper');
const json = $('script[type="application/ld+json"]').html(); const json = $('script[type="application/ld+json"]').html();
const videoJson = $('script:contains("ScenePlayerOptions")').html();
const videoDataString = videoJson.slice(videoJson.indexOf('= {') + 2, videoJson.indexOf('};') + 1);
const data = JSON.parse(json)[0]; const data = JSON.parse(json)[0];
const videoData = JSON.parse(videoDataString);
const entryId = new URL(url).pathname.split('/').slice(-1)[0]; const entryId = new URL(url).pathname.split('/').slice(-1)[0];
const title = data.isPartOf ? data.isPartOf.name : data.name; const title = videoData?.playerOptions?.sceneInfos?.sceneTitle || (data.isPartOf && data.isPartOf !== 'TBD' ? data.isPartOf.name : data.name);
const dataDate = moment.utc(data.dateCreated, 'YYYY-MM-DD'); const dataDate = moment.utc(videoData?.playerOptions?.sceneInfos?.sceneReleaseDate, 'YYYY-MM-DD');
const date = dataDate.isValid() const date = dataDate.isValid()
? dataDate.toDate() ? dataDate.toDate()
: moment.utc(sceneElement.find('.updatedDate').text().trim(), 'MM-DD-YYYY').toDate(); : moment.utc(sceneElement.find('.updatedDate').text().trim(), 'MM-DD-YYYY').toDate();
const actors = data.actor const actors = data.actor.map(actor => actor.name);
.sort(({ gender: genderA }, { gender: genderB }) => {
if (genderA === 'female' && genderB === 'male') return -1;
if (genderA === 'male' && genderB === 'female') return 1;
return 0;
})
.map(actor => actor.name);
const description = data.description || null; // prevent empty string const description = data.description || null; // prevent empty string
const likes = Number(sceneElement.find('.rating .state_1 .value').text()); const likes = Number(sceneElement.find('.rating .state_1 .value').text());
@ -84,27 +128,18 @@ async function scrapeScene(html, url, site) {
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds(); const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
const rawTags = data.keywords.split(', '); const poster = videoData.picPreview;
const trailer = `${videoData.playerOptions.host}${videoData.url}`;
const photoPath = $('.picturesItem a').attr('href');
const photos = await getPhotos(photoPath, site);
const tags = data.keywords.split(', ');
const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title'); const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
const siteId = siteName && siteName.replace(/\s+/g, '').toLowerCase(); const channel = siteName && siteName.replace(/\s+/g, '').toLowerCase();
const [channelSite, tags] = await Promise.all([
site.isFallback
? knex('sites')
.where({ slug: siteId })
.orWhereRaw('name = ? collate NOCASE', [siteName])
.first()
: site,
matchTags(rawTags),
]);
// only replace generic URL with site URL if site is not marked to fetch scenes from generic site
const originalUrl = channelSite && !(channelSite.parameters && JSON.parse(channelSite.parameters).filter)
? `${channelSite.url}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`
: url;
return { return {
url: originalUrl, url,
entryId, entryId,
title, title,
date, date,
@ -112,22 +147,30 @@ async function scrapeScene(html, url, site) {
description, description,
duration, duration,
tags, tags,
poster,
photos,
trailer: {
src: trailer,
},
rating: { rating: {
likes, likes,
dislikes, dislikes,
}, },
site: channelSite || site, site,
channel,
}; };
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.parameters && site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`); const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`;
const res = await bhttp.get(url);
return scrape(res.body.toString(), site); return scrape(res.body.toString(), site);
} }
async function fetchUpcoming(site) { async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.parameters && site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`); const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`;
const res = await bhttp.get(url);
return scrape(res.body.toString(), site); return scrape(res.body.toString(), site);
} }

View File

@ -6,7 +6,6 @@ const { JSDOM } = require('jsdom');
const moment = require('moment'); const moment = require('moment');
const knex = require('../knex'); const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
function scrapeLatest(html, site) { function scrapeLatest(html, site) {
@ -49,13 +48,16 @@ async function scrapeScene(html, url, site) {
const title = $('meta[itemprop="name"]').attr('content'); const title = $('meta[itemprop="name"]').attr('content');
const description = $('.descr-box p').text(); // meta tags don't contain full description const description = $('.descr-box p').text(); // meta tags don't contain full description
const date = moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate(); const dateProp = $('meta[itemprop="uploadDate"]').attr('content');
const date = dateProp
? moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate()
: moment.utc($('.title-border:nth-child(2) p').text(), 'MM.DD.YYYY').toDate();
const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray(); const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($('.info-panel.likes .likes').text()); const likes = Number($('.info-panel.likes .likes').text());
const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60; const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
const rawTags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const tags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const poster = $('#video').attr('poster'); const poster = $('#video').attr('poster');
const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray(); const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
@ -63,21 +65,7 @@ async function scrapeScene(html, url, site) {
const trailer540 = $('source[res="540"]').attr('src'); const trailer540 = $('source[res="540"]').attr('src');
const trailer720 = $('source[res="720"]').attr('src'); const trailer720 = $('source[res="720"]').attr('src');
/*
* broken as of nov 2019
const { origin } = new URL($('.pornstar-card meta[itemprop="url"]').first().attr('content'));
const [channelSite, tags] = await Promise.all([
// don't find site if original is already specific
site.isFallback ? knex('sites').where({ url: origin }).first() : site,
matchTags(rawTags),
]);
*/
const tags = await matchTags(rawTags);
return { return {
// url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
url, url,
entryId, entryId,
title, title,
@ -88,20 +76,19 @@ async function scrapeScene(html, url, site) {
tags, tags,
poster, poster,
photos, photos,
trailer: trailer540 trailer: [
? { {
src: trailer540,
quality: 540,
}
: {
// backup
src: trailer720, src: trailer720,
quality: 720, quality: 720,
}, },
{
src: trailer540,
quality: 540,
},
],
rating: { rating: {
likes, likes,
}, },
// site: channelSite || site,
site, site,
}; };
} }

View File

@ -8,7 +8,6 @@ const moment = require('moment');
const knex = require('../knex'); const knex = require('../knex');
const { matchTags } = require('../tags'); const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
async function getPhoto(url) { async function getPhoto(url) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
@ -20,7 +19,7 @@ async function getPhoto(url) {
return photoUrl; return photoUrl;
} }
async function getPhotos(albumUrl, site, siteUrl) { async function getPhotos(albumUrl) {
const res = await bhttp.get(albumUrl); const res = await bhttp.get(albumUrl);
const html = res.body.toString(); const html = res.body.toString();
const { document } = new JSDOM(html).window; const { document } = new JSDOM(html).window;
@ -28,15 +27,7 @@ async function getPhotos(albumUrl, site, siteUrl) {
const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href; const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10); const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
// dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween const photoUrls = await Promise.map(Array.from({ length: lastPhotoIndex }), async (index) => {
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);
if (photoLimit > 25) {
console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
}
const photoUrls = await Promise.map(photoIndexes, async (index) => {
const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`; const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`;
return getPhoto(pageUrl); return getPhoto(pageUrl);

View File

@ -9,8 +9,6 @@ const moment = require('moment');
const { heightToCm } = require('../utils/convert'); const { heightToCm } = require('../utils/convert');
const { matchTags } = require('../tags'); const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
async function fetchPhotos(url) { async function fetchPhotos(url) {
const res = await bhttp.get(url); const res = await bhttp.get(url);
@ -58,14 +56,7 @@ async function getPhotos(entryId, site, page = 1) {
}) })
: []; : [];
const allPhotos = photos.concat(otherPhotos.flat()); return photos.concat(otherPhotos.flat());
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
return pluckedPhotos;
} }
function scrapeLatest(html, site) { function scrapeLatest(html, site) {

View File

@ -8,7 +8,6 @@ const moment = require('moment');
const { fetchSites } = require('../sites'); const { fetchSites } = require('../sites');
const { matchTags } = require('../tags'); const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');
const defaultTags = { const defaultTags = {
hardx: [], hardx: [],
@ -38,7 +37,7 @@ function scrapePhotos(html) {
return unlockedPhotos.concat(lockedThumbnails); return unlockedPhotos.concat(lockedThumbnails);
} }
async function getPhotos(albumPath, siteDomain, site) { async function getPhotos(albumPath, siteDomain) {
const albumUrl = `https://${siteDomain}${albumPath}`; const albumUrl = `https://${siteDomain}${albumPath}`;
const html = await fetchPhotos(albumUrl); const html = await fetchPhotos(albumUrl);
@ -56,14 +55,7 @@ async function getPhotos(albumPath, siteDomain, site) {
concurrency: 2, concurrency: 2,
}); });
const allPhotos = photos.concat(otherPhotos.flat()); return photos.concat(otherPhotos.flat());
const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
return pluckedPhotos;
} }
function scrape(html, site) { function scrape(html, site) {

View File

@ -1,13 +1,18 @@
'use strict'; 'use strict';
const config = require('config');
// pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1 // pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
function pluckPhotos(photoTotal, photoLimit) { function pluckPhotos(photos, release, specifiedLimit) {
const limit = specifiedLimit || config.media.limit;
console.log(limit);
const plucked = [1] const plucked = [1]
.concat( .concat(
Array.from({ length: photoLimit - 1 }, (value, index) => Math.round((index + 1) * (photoTotal / (photoLimit - 1)))), Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (photos.length / (limit)))),
); );
return Array.from(new Set(plucked)); // remove duplicates, may happen when photo total and photo limit are close return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex]); // remove duplicates, may happen when photo total and photo limit are close
} }
module.exports = pluckPhotos; module.exports = pluckPhotos;

2
traxxx
View File

@ -1,2 +1,2 @@
#!/usr/bin/bash #!/usr/bin/bash
node ./src/app.js "$@"; node ./dist/init.js "$@";