Refactored 21sextury scraper.

.babelrc

@@ -3,5 +3,6 @@
         [
             "@babel/preset-env"
         ]
-    ]
+    ],
+    "plugins": ["@babel/plugin-proposal-optional-chaining"]
 }

.eslintrc

@@ -2,6 +2,7 @@
     "root": true,
     "extends": ["airbnb-base", "plugin:vue/recommended"],
     "parserOptions": {
+        "parser": "babel-eslint",
         "ecmaVersion": 2019,
         "sourceType": "module"
     },

assets/components/release/release.vue

@@ -293,6 +293,7 @@ export default {
 
 .logo {
     display: inline-block;
+    filter: drop-shadow(0 0 1px $shadow);
 }
 
 .logo-site {
@@ -300,6 +301,7 @@ export default {
     max-width: 15rem;
     object-fit: contain;
     object-position: 100% 50%;
+    filter: drop-shadow(0 0 1px $shadow);
 }
 
 .logo-network {

assets/components/tags/tag.vue

@@ -9,7 +9,7 @@
             <div class="sidebar">
                 <a
                     v-if="tag.poster"
-                    :href="`/media/${tag.poster.path}`"
+                    :href="`/img/${tag.poster.path}`"
                     :title="tag.poster.comment"
                     target="_blank"
                     rel="noopener noreferrer"

assets/components/tags/tags.vue

@@ -98,6 +98,8 @@ export default {
 </script>
 
 <style lang="scss" scoped>
+@import 'theme';
+
 .tags {
     padding: 1rem;
 }

assets/components/tile/tag.vue

@@ -30,8 +30,8 @@ export default {
 @import 'theme';
 
 .tile {
-    color: $text-contrast;
+    color: $text;
-    background: $profile;
+    background: $background;
     display: flex;
     flex-direction: column;
     align-items: center;

config/default.js

@@ -86,5 +86,6 @@ module.exports = {
     media: {
         path: './',
         thumbnailSize: 320, // width for 16:9 will be exactly 576px
+        limit: 25, // max number of photos per release
     },
 };

package.json

@@ -5,10 +5,12 @@
     "main": "src/app.js",
     "scripts": {
         "postinstall": "npm run migrate && npm run seed",
-        "start": "node src/app.js",
+        "start": "node dist/init.js",
         "webpack": "webpack --env=production --mode=production",
         "webpack-dev": "webpack --env=development --mode=development",
         "webpack-watch": "webpack --progress --colors --watch --env=development --mode=development",
+        "babel": "babel src -d dist",
+        "babel-watch": "babel src -w -d dist",
         "eslint": "eslint src/",
         "eslint-watch": "esw --watch src/",
         "knex": "knex",
@@ -32,68 +34,70 @@
     "author": "Niels Simenon",
     "license": "ISC",
     "devDependencies": {
-        "@babel/cli": "^7.2.3",
+        "@babel/cli": "^7.7.5",
-        "@babel/core": "^7.4.5",
+        "@babel/core": "^7.7.5",
-        "@babel/preset-env": "^7.4.5",
+        "@babel/plugin-proposal-optional-chaining": "^7.7.5",
-        "autoprefixer": "^9.5.1",
+        "@babel/preset-env": "^7.7.6",
+        "autoprefixer": "^9.7.3",
         "babel-cli": "^6.26.0",
-        "babel-eslint": "^10.0.1",
+        "babel-eslint": "^10.0.3",
         "babel-loader": "^8.0.6",
-        "babel-preset-airbnb": "^3.2.0",
+        "babel-preset-airbnb": "^3.3.2",
         "babel-register": "^6.26.0",
         "css-loader": "^2.1.1",
         "eslint": "^5.16.0",
-        "eslint-config-airbnb": "^17.1.0",
+        "eslint-config-airbnb": "^17.1.1",
-        "eslint-config-airbnb-base": "^13.1.0",
+        "eslint-config-airbnb-base": "^13.2.0",
-        "eslint-loader": "^2.1.2",
+        "eslint-loader": "^2.2.1",
-        "eslint-plugin-import": "^2.17.3",
+        "eslint-plugin-import": "^2.18.2",
-        "eslint-plugin-jsx-a11y": "^6.2.1",
+        "eslint-plugin-jsx-a11y": "^6.2.3",
-        "eslint-plugin-react": "^7.13.0",
+        "eslint-plugin-react": "^7.17.0",
-        "eslint-plugin-vue": "^5.2.2",
+        "eslint-plugin-vue": "^6.0.1",
         "eslint-watch": "^4.0.2",
         "mini-css-extract-plugin": "^0.7.0",
-        "node-sass": "^4.12.0",
+        "node-sass": "^4.13.0",
         "postcss-loader": "^3.0.0",
         "raw-loader": "^2.0.0",
-        "sass-loader": "^7.1.0",
+        "sass-loader": "^7.3.1",
         "style-loader": "^0.23.1",
-        "vue-loader": "^15.7.0",
+        "vue-loader": "^15.7.2",
         "vue-template-compiler": "^2.6.10",
-        "webpack": "^4.32.2",
+        "webpack": "^4.41.2",
-        "webpack-cli": "^3.3.2"
+        "webpack-cli": "^3.3.10"
     },
     "dependencies": {
+        "babel-polyfill": "^6.26.0",
         "bhttp": "^1.2.4",
         "blake2": "^4.0.0",
-        "bluebird": "^3.5.4",
+        "bluebird": "^3.7.2",
         "body-parser": "^1.19.0",
-        "cheerio": "^1.0.0-rc.2",
+        "cheerio": "^1.0.0-rc.3",
         "cli-confirm": "^1.0.1",
-        "config": "^3.0.1",
+        "config": "^3.2.4",
-        "dayjs": "^1.8.14",
+        "dayjs": "^1.8.17",
-        "express": "^4.16.4",
+        "express": "^4.17.1",
         "express-promise-router": "^3.0.3",
         "express-react-views": "^0.11.0",
         "fs-extra": "^7.0.1",
-        "jsdom": "^15.2.0",
+        "jsdom": "^15.2.1",
-        "knex": "^0.16.3",
+        "knex": "^0.16.5",
-        "knex-migrate": "^1.7.1",
+        "knex-migrate": "^1.7.4",
         "mime": "^2.4.4",
         "moment": "^2.24.0",
-        "opn": "^5.4.0",
+        "opn": "^5.5.0",
-        "pg": "^7.9.0",
+        "pg": "^7.14.0",
         "prop-types": "^15.7.2",
-        "react": "^16.8.6",
+        "react": "^16.12.0",
-        "react-dom": "^16.8.6",
+        "react-dom": "^16.12.0",
-        "sharp": "^0.23.2",
+        "sharp": "^0.23.4",
         "showdown": "^1.9.1",
         "tough-cookie": "^3.0.1",
-        "tty-table": "^2.7.0",
+        "tty-table": "^2.8.3",
         "url-pattern": "^1.0.3",
         "v-tooltip": "^2.0.2",
         "vue": "^2.6.10",
-        "vue-router": "^3.0.6",
+        "vue-router": "^3.1.3",
-        "vuex": "^3.1.1",
+        "vuex": "^3.1.2",
-        "yargs": "^13.2.2"
+        "yargs": "^13.3.0"
     }
 }

public/css/style.css

@@ -42,7 +42,9 @@
 /* $primary: #ff886c; */
 .filter-bar[data-v-6db17c96] {
   background: #fff;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-pack: justify;
           justify-content: space-between;
   padding: .5rem 1rem;
   z-index: 1;
@@ -93,7 +95,10 @@
 /* $primary: #ff886c; */
 .tile[data-v-3abcf101] {
   background: #fff;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
   box-sizing: border-box;
   padding: 0 0 .5rem 0;
@@ -108,8 +113,11 @@
 .thumbnail[data-v-3abcf101] {
   width: 100%;
   height: 12rem;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-pack: center;
           justify-content: center;
+  -webkit-box-align: center;
           align-items: center;
   -o-object-fit: cover;
      object-fit: cover;
@@ -120,8 +128,11 @@
   text-shadow: 1px 1px 0 rgba(255, 255, 255, 0.5);
 }
 .row[data-v-3abcf101] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-pack: justify;
           justify-content: space-between;
+  -webkit-box-align: center;
           align-items: center;
   box-sizing: border-box;
   padding: 0 .5rem;
@@ -129,8 +140,11 @@
 }
 .details[data-v-3abcf101] {
   width: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: justify;
           justify-content: space-between;
   position: absolute;
   font-size: 0;
@@ -162,8 +176,12 @@
   font-weight: bold;
 }
 .info[data-v-3abcf101] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-flex: 1;
           flex-grow: 1;
 }
 .link[data-v-3abcf101] {
@@ -281,8 +299,11 @@
   background: rgba(0, 0, 0, 0.1);
   height: 12rem;
   width: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   -o-object-fit: cover;
      object-fit: cover;
@@ -324,7 +345,9 @@
   box-sizing: border-box;
 }
 .row[data-v-2bc41e74] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
   margin: 0 0 1rem 0;
 }
@@ -341,7 +364,9 @@
   cursor: default;
 }
 .info .column[data-v-2bc41e74] {
+    display: -webkit-box;
     display: flex;
+    -webkit-box-align: center;
             align-items: center;
     padding: 0 1rem;
 }
@@ -362,15 +387,21 @@
     margin: 0 1rem 0 0;
 }
 .site[data-v-2bc41e74] {
+  display: -webkit-inline-box;
   display: inline-flex;
+  -webkit-box-flex: 1;
           flex-grow: 1;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: end;
           justify-content: flex-end;
   padding: .25rem 0;
   font-size: 0;
 }
 .logo[data-v-2bc41e74] {
   display: inline-block;
+  -webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
+          filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
 }
 .logo-site[data-v-2bc41e74] {
   height: 3rem;
@@ -379,6 +410,8 @@
      object-fit: contain;
   -o-object-position: 100% 50%;
      object-position: 100% 50%;
+  -webkit-filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
+          filter: drop-shadow(0 0 1px rgba(0, 0, 0, 0.5));
 }
 .logo-network[data-v-2bc41e74] {
   height: 1.5rem;
@@ -407,6 +440,7 @@
   font-size: 1rem;
 }
 .actors[data-v-2bc41e74] {
+  display: -webkit-box;
   display: flex;
   flex-wrap: wrap;
 }
@@ -460,12 +494,16 @@
 
 /* $primary: #ff886c; */
 .header[data-v-3e57cf44] {
+  display: -webkit-box;
   display: flex;
   flex-wrap: wrap;
+  -webkit-box-pack: justify;
           justify-content: space-between;
 }
 .title[data-v-3e57cf44] {
+  display: -webkit-inline-box;
   display: inline-flex;
+  -webkit-box-align: top;
           align-items: top;
   margin: 0 1rem 0 0;
 }
@@ -477,9 +515,13 @@
   margin: 0 0 1rem 0;
 }
 .link[data-v-3e57cf44] {
+  display: -webkit-box;
   display: flex;
   flex-shrink: 0;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-align: end;
           align-items: flex-end;
 }
 .logo[data-v-3e57cf44] {
@@ -490,7 +532,9 @@
   margin: 0 .5rem 1rem 0;
 }
 .networklogo-container[data-v-3e57cf44] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
 }
 .networklogo[data-v-3e57cf44] {
@@ -517,8 +561,12 @@
 /* $primary: #ff886c; */
 .tile[data-v-f4958086] {
   background: #fff;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-align: center;
           align-items: center;
   box-sizing: border-box;
   padding: .5rem 1rem;
@@ -534,8 +582,11 @@
   width: 100%;
   height: 5rem;
   color: #222;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   -o-object-fit: contain;
      object-fit: contain;
@@ -547,21 +598,28 @@
 .title[data-v-f4958086] {
   color: #222;
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
   margin: 0;
 }
 
 /* $primary: #ff886c; */
 .header[data-v-e2e12602] {
+  display: -webkit-box;
   display: flex;
   flex-wrap: wrap;
+  -webkit-box-pack: justify;
           justify-content: space-between;
+  -webkit-box-align: top;
           align-items: top;
   margin: 0 0 2rem 0;
 }
 .title[data-v-e2e12602] {
+  display: -webkit-inline-box;
   display: inline-flex;
+  -webkit-box-align: top;
           align-items: top;
   margin: 0 1rem 0 0;
 }
@@ -592,8 +650,12 @@
 /* $primary: #ff886c; */
 .tile[data-v-8b4c90b0] {
   background: #fff;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-align: center;
           align-items: center;
   box-sizing: border-box;
   padding: .5rem 1rem;
@@ -609,8 +671,11 @@
   width: 100%;
   height: 5rem;
   color: #222;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   -o-object-fit: contain;
      object-fit: contain;
@@ -622,7 +687,9 @@
 .title[data-v-8b4c90b0] {
   color: #222;
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
   margin: 0;
 }
@@ -660,6 +727,7 @@
 .photos[data-v-0a0430c7] {
     width: 100%;
     max-width: 100%;
+    display: -webkit-box;
     display: flex;
     overflow-x: scroll;
     scrollbar-width: none;
@@ -680,7 +748,10 @@
 /* $primary: #ff886c; */
 .actor-inner[data-v-ea0483c2] {
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
   padding: 0;
   overflow-x: auto;
@@ -689,7 +760,10 @@
   background: #222;
   color: rgba(255, 255, 255, 0.9);
   width: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: horizontal;
+  -webkit-box-direction: normal;
           flex-direction: row;
   flex-shrink: 0;
 }
@@ -708,6 +782,7 @@
        object-position: 50% 0;
 }
 .bio[data-v-ea0483c2] {
+  -webkit-box-flex: 1;
           flex-grow: 1;
   min-width: 20rem;
   box-sizing: border-box;
@@ -715,13 +790,18 @@
   margin: 0 2rem 0 0;
 }
 .bio-header[data-v-ea0483c2] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-pack: justify;
           justify-content: space-between;
+  -webkit-box-align: center;
           align-items: center;
   margin: 0 0 1rem 0;
 }
 .bio-item[data-v-ea0483c2] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-pack: justify;
           justify-content: space-between;
   padding: 0 0 .25rem 0;
   margin: 0 0 .25rem 0;
@@ -801,6 +881,7 @@
   font-size: .8rem;
 }
 .extra[data-v-ea0483c2] {
+  -webkit-box-flex: 1;
           flex-grow: 1;
 }
 .description[data-v-ea0483c2] {
@@ -837,8 +918,12 @@
     fill: #ff6c88;
 }
 .actor-content[data-v-ea0483c2] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-flex: 1;
           flex-grow: 1;
+  -webkit-box-orient: horizontal;
+  -webkit-box-direction: normal;
           flex-direction: row;
 }
 .heading[data-v-ea0483c2] {
@@ -856,6 +941,7 @@
   display: none;
 }
 .releases[data-v-ea0483c2] {
+  -webkit-box-flex: 1;
           flex-grow: 1;
   padding: 1rem;
 }
@@ -870,6 +956,8 @@
     display: none;
 }
 .actor-content[data-v-ea0483c2] {
+    -webkit-box-orient: vertical;
+    -webkit-box-direction: normal;
             flex-direction: column;
 }
 .photos-container[data-v-ea0483c2] {
@@ -882,11 +970,14 @@
     display: none;
 }
 .photos.compact[data-v-ea0483c2] {
+    display: -webkit-box;
     display: flex;
 }
 }
 @media (max-width: 720px) {
 .profile[data-v-ea0483c2] {
+    -webkit-box-orient: vertical;
+    -webkit-box-direction: normal;
             flex-direction: column;
     padding: 0 0 .5rem 0;
 }
@@ -931,9 +1022,14 @@
 
 /* $primary: #ff886c; */
 .tag[data-v-7f130e7f] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: horizontal;
+  -webkit-box-direction: normal;
           flex-direction: row;
+  -webkit-box-flex: 1;
           flex-grow: 1;
+  -webkit-box-pack: stretch;
           justify-content: stretch;
 }
 .sidebar[data-v-7f130e7f] {
@@ -970,10 +1066,14 @@
 
 /* $primary: #ff886c; */
 .tile[data-v-602c6fd8] {
-  color: #fff;
+  color: #222;
-  background: #222;
+  background: #fff;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-align: center;
           align-items: center;
   box-sizing: border-box;
   box-shadow: 0 0 3px rgba(0, 0, 0, 0.25);
@@ -988,8 +1088,11 @@
 }
 .title[data-v-602c6fd8] {
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   font-size: 1rem;
   padding: .5rem 1rem;
@@ -997,6 +1100,7 @@
   text-transform: capitalize;
 }
 
+/* $primary: #ff886c; */
 .tags[data-v-66fa6284] {
   padding: 1rem;
 }
@@ -1011,9 +1115,14 @@
   background: #fff;
   color: #ff6c88;
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   font-size: 2rem;
 }
@@ -1149,7 +1258,9 @@ body {
 
 /* $primary: #ff886c; */
 .header[data-v-10b7ec04] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
   background: #fff;
   color: #ff6c88;
@@ -1171,8 +1282,11 @@ body {
   display: inline-block;
 }
 .nav-link[data-v-10b7ec04] {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-align: center;
           align-items: center;
+  -webkit-box-pack: center;
           justify-content: center;
   padding: 1rem;
   border-bottom: solid 5px transparent;
@@ -1203,10 +1317,12 @@ body {
     display: none;
 }
 .nav .nolist[data-v-10b7ec04] {
+    display: -webkit-box;
     display: flex;
 }
 .nav[data-v-10b7ec04],
   .nav-item[data-v-10b7ec04] {
+    -webkit-box-flex: 1;
             flex-grow: 1;
 }
 }
@@ -1215,17 +1331,25 @@ body {
 .container {
   background: #fafafa;
   height: 100%;
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
   overflow: hidden;
 }
 .content {
+  display: -webkit-box;
   display: flex;
+  -webkit-box-orient: vertical;
+  -webkit-box-direction: normal;
           flex-direction: column;
+  -webkit-box-flex: 1;
           flex-grow: 1;
   overflow-y: auto;
 }
 .content-inner {
+  -webkit-box-flex: 1;
           flex-grow: 1;
   padding: 1rem;
   overflow-y: auto;

seeds/01_sites.js

@@ -21,23 +21,17 @@ function getSites(networksMap) {
         {
             slug: 'buttplays',
             name: 'Butt Plays',
-            url: 'https://www.buttplays.com',
             network_id: networksMap['21sextury'],
-            parameters: JSON.stringify({ filter: true }),
         },
         {
             slug: 'clubsandy',
             name: 'Club Sandy',
-            url: 'https://www.clubsandy.com',
             network_id: networksMap['21sextury'],
-            parameters: JSON.stringify({ filter: true }),
         },
         {
             slug: 'deepthroatfrenzy',
             name: 'Deepthroat Frenzy',
-            url: 'https://www.deepthroatfrenzy.com',
             network_id: networksMap['21sextury'],
-            parameters: JSON.stringify({ filter: true }),
         },
         {
             slug: 'dpfanatics',
@@ -56,9 +50,7 @@ function getSites(networksMap) {
         {
             slug: 'gapeland',
             name: 'Gapeland',
-            url: 'https://www.gapeland.com',
             network_id: networksMap['21sextury'],
-            parameters: JSON.stringify({ filter: true }),
         },
         {
             slug: 'lezcuties',
@@ -67,12 +59,65 @@ function getSites(networksMap) {
             description: 'LezCuties brings you the cutest lesbian coeds and tiny teen lesbians in HD lesbian porn. Watch as European teens explore themselves and lick each other\'s tight lesbian pussy while their parents aren\'t home.',
             network_id: networksMap['21sextury'],
         },
+        {
+            slug: 'onlyswallows',
+            name: 'Only Swallows',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'alettaoceanempire',
+            name: 'Aletta Ocean Empire',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'analqueenalysa',
+            name: 'Anal Queen Alysa',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'blueangellive',
+            name: 'Blue Angel Live',
+            network_id: networksMap['21sextury'],
+        },
         {
             slug: 'pixandvideo',
             name: 'Pix and Video',
-            url: 'https://www.pixandvideo.com',
             network_id: networksMap['21sextury'],
-            parameters: JSON.stringify({ filter: true }),
+        },
+        {
+            slug: 'cheatingwhorewives',
+            name: 'Cheating Whore Wives',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'cutiesgalore',
+            name: 'Cuties Galore',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'hotmilfclub',
+            name: 'Hot MILF Club',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'letsplaylez',
+            name: 'Lets Play Lez',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'nudefightclub',
+            name: 'Nude Fight Club',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'sexwithkathianobili',
+            name: 'Sex With Kathia Nobili',
+            network_id: networksMap['21sextury'],
+        },
+        {
+            slug: 'sweetsophiemoone',
+            name: 'Sweet Sophie Moone',
+            network_id: networksMap['21sextury'],
         },
         // BANGBROS
         {

seeds/04_media.js

@@ -72,6 +72,12 @@ function getMedia(tagsMap) {
             role: 'poster',
             comment: 'Juelz Ventura in "Gangbanged 5" for Elegant Angel',
         },
+        {
+            path: 'tags/tattoo.jpeg',
+            target_id: tagsMap.tattoo,
+            role: 'poster',
+            comment: 'Kali Roses in "Goes All In For Anal" for Hussie Pass',
+        },
         {
             path: 'tags/triple-anal.jpeg',
             target_id: tagsMap['triple-anal'],

src/.eslintrc

@@ -1,13 +1,15 @@
 {
     "extends": "airbnb-base",
     "parserOptions": {
+        "parser": "babel-eslint",
         "sourceType": "script"
     },
     "rules": {
         "strict": 0,
         "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
         "no-console": 0,
-        "indent": ["error", 4],
+        "indent": "off",
-        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
+        "template-curly-spacing": "off"
     }
 }

src/actors.js

@@ -271,6 +271,8 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
 }
 
 async function mergeProfiles(profiles, actor) {
+    console.log(profiles);
+
     const mergedProfile = profiles.reduce((prevProfile, profile) => {
         if (profile === null) {
             return prevProfile;

src/app.js

@@ -46,4 +46,4 @@ async function init() {
     await initServer();
 }
 
-init();
+module.exports = init;

src/argv.js

@@ -63,6 +63,7 @@ const { argv } = yargs
     .option('debug', {
         describe: 'Show error stack traces',
         type: 'boolean',
+        default: process.env.NODE_ENV === 'development',
     });
 
 module.exports = argv;

src/init.js

@@ -0,0 +1,4 @@
+require('babel-polyfill');
+const init = require('./app');
+
+init();

src/media.js

@@ -10,6 +10,7 @@ const sharp = require('sharp');
 const blake2 = require('blake2');
 
 const knex = require('./knex');
+const pluckPhotos = require('./utils/pluck-photos');
 
 function getHash(buffer) {
     const hash = blake2.createHash('blake2b', { digestLength: 24 });
@@ -94,10 +95,10 @@ async function filterHashDuplicates(files, domains = ['releases'], roles = ['pho
 }
 
 async function fetchPhoto(photoUrl, index, identifier) {
+    try {
         const { pathname } = new URL(photoUrl);
         const mimetype = mime.getType(pathname);
 
-    try {
         const res = await bhttp.get(photoUrl);
 
         if (res.statusCode === 200) {
@@ -176,7 +177,11 @@ async function storePhotos(release, releaseId) {
         return;
     }
 
-    const newPhotos = await filterSourceDuplicates(release.photos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
+    const pluckedPhotos = pluckPhotos(release.photos, release);
+
+    console.log(release.photos, pluckedPhotos);
+
+    const newPhotos = await filterSourceDuplicates(pluckedPhotos, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
 
     if (newPhotos.length === 0) return;
 

src/releases.js

@@ -125,7 +125,7 @@ async function curateScrapedRelease(release) {
         likes: release.rating && release.rating.likes,
         dislikes: release.rating && release.rating.dislikes,
         rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
-        deep: Boolean(argv.deep && release.url && !release.upcoming),
+        deep: typeof release.deep === 'boolean' ? release.deep : false,
     };
 
     if (release.site.isFallback && release.channel) {
@@ -275,6 +275,12 @@ async function storeRelease(release) {
 
 async function storeReleases(releases) {
     const storedReleases = await Promise.map(releases, async (release) => {
+        if (release.site.isFallback && !release.channel) {
+            console.error(`Unable to derive channel site from generic URL: ${release.url}.`);
+
+            return null;
+        }
+
         try {
             const releaseId = await storeRelease(release);
 
@@ -289,7 +295,7 @@ async function storeReleases(releases) {
         }
     }, {
         concurrency: 10,
-    });
+    }).filter(release => release);
 
     const actors = storedReleases.reduce((acc, release) => {
         if (!release.actors) return acc;

src/scrape-release.js

@@ -51,8 +51,10 @@ async function scrapeRelease(url, release, deep = false) {
         // don't store release when called by site scraper
         const [storedRelease] = await storeReleases([scene]);
 
+        if (storedRelease) {
             console.log(`http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`);
         }
+    }
 
     return scene;
 }

src/scrape-sites.js

@@ -69,12 +69,20 @@ async function scrapeUpcomingReleases(scraper, site) {
 async function deepFetchReleases(baseReleases) {
     return Promise.map(baseReleases, async (release) => {
         if (release.url) {
+            try {
                 const fullRelease = await scrapeRelease(release.url, release, true);
 
                 return {
                     ...release,
                     ...fullRelease,
+                    deep: true,
                 };
+            } catch (error) {
+                return {
+                    ...release,
+                    deep: false,
+                };
+            }
         }
 
         return release;
@@ -116,7 +124,7 @@ async function scrapeReleases() {
             return await scrapeSiteReleases(scraper, site);
         } catch (error) {
             if (argv.debug) {
-                console.error(`${site.id}: Failed to scrape releases`, error);
+                console.error(`${site.name}: Failed to scrape releases`, error);
             }
 
             console.warn(`${site.id}: Failed to scrape releases`);

src/scrapers/21sextury.js

@@ -1,11 +1,51 @@
 'use strict';
 
+const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const knex = require('../knex');
+async function fetchPhotos(photoPath) {
-const { matchTags } = require('../tags');
+    const res = await bhttp.get(`https://21sextury.com${photoPath}`);
+
+    return res.body.toString();
+}
+
+function scrapePhotos(html) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+
+    const unlockedPhotos = $('.preview .imgLink.pgUnlocked')
+        .map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
+
+    const lockedThumbnails = $('.preview .imgLink.lockedPicture img')
+        .map((photoIndex, photoElement) => $(photoElement)
+            .attr('src'))
+        // .replace('_tb.jpg', '.jpg')) does not always work
+        .toArray();
+
+    return unlockedPhotos.concat(lockedThumbnails);
+}
+
+async function getPhotos(photoPath) {
+    if (!photoPath || photoPath.match('join')) {
+        return [];
+    }
+
+    const html = await fetchPhotos(photoPath);
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const photos = scrapePhotos(html);
+    const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray();
+
+    const otherPhotos = await Promise.map(pages, async (pagePath) => {
+        const pageHtml = await fetchPhotos(pagePath);
+
+        return scrapePhotos(pageHtml);
+    }, {
+        concurrency: 2,
+    });
+
+    return photos.concat(otherPhotos.flat());
+}
 
 function scrape(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -14,12 +54,13 @@ function scrape(html, site) {
     return scenesElements.reduce((accReleases, element) => {
         const siteName = $(element).find('.studioName a').attr('title');
 
-        if (site.parameters && site.parameters.filter && siteName.toLowerCase() !== site.name.toLowerCase()) {
+        if (!site.url && siteName.toLowerCase() !== site.name.toLowerCase()) {
+            // using generic overview as fallback, scene from different site
             return accReleases;
         }
 
         const sceneLinkElement = $(element).find('.sceneTitle a');
-        const url = `${site.url}${sceneLinkElement.attr('href')}`;
+        const url = `${site.url || 'https://www.21sextury.com'}${sceneLinkElement.attr('href')}`;
         const title = sceneLinkElement.attr('title').trim();
 
         const entryId = $(element).attr('data-itemid');
@@ -32,6 +73,9 @@ function scrape(html, site) {
             .map((actorIndex, actorElement) => $(actorElement).attr('title'))
             .toArray();
 
+        const poster = $(element).find('.imgLink img').attr('data-original');
+        const trailer = `https://videothumb.gammacdn.com/307x224/${entryId}.mp4`;
+
         const [likes, dislikes] = $(element).find('.value')
             .toArray()
             .map(value => Number($(value).text()));
@@ -44,6 +88,10 @@ function scrape(html, site) {
                 title,
                 actors,
                 date,
+                poster,
+                trailer: {
+                    src: trailer,
+                },
                 rating: {
                     likes,
                     dislikes,
@@ -58,25 +106,21 @@ async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
     const sceneElement = $('#videoWrapper');
     const json = $('script[type="application/ld+json"]').html();
+    const videoJson = $('script:contains("ScenePlayerOptions")').html();
+    const videoDataString = videoJson.slice(videoJson.indexOf('= {') + 2, videoJson.indexOf('};') + 1);
 
     const data = JSON.parse(json)[0];
+    const videoData = JSON.parse(videoDataString);
     const entryId = new URL(url).pathname.split('/').slice(-1)[0];
 
-    const title = data.isPartOf ? data.isPartOf.name : data.name;
+    const title = videoData?.playerOptions?.sceneInfos?.sceneTitle || (data.isPartOf && data.isPartOf !== 'TBD' ? data.isPartOf.name : data.name);
-    const dataDate = moment.utc(data.dateCreated, 'YYYY-MM-DD');
+    const dataDate = moment.utc(videoData?.playerOptions?.sceneInfos?.sceneReleaseDate, 'YYYY-MM-DD');
 
     const date = dataDate.isValid()
         ? dataDate.toDate()
         : moment.utc(sceneElement.find('.updatedDate').text().trim(), 'MM-DD-YYYY').toDate();
 
-    const actors = data.actor
+    const actors = data.actor.map(actor => actor.name);
-        .sort(({ gender: genderA }, { gender: genderB }) => {
-            if (genderA === 'female' && genderB === 'male') return -1;
-            if (genderA === 'male' && genderB === 'female') return 1;
-
-            return 0;
-        })
-        .map(actor => actor.name);
 
     const description = data.description || null; // prevent empty string
     const likes = Number(sceneElement.find('.rating .state_1 .value').text());
@@ -84,27 +128,18 @@ async function scrapeScene(html, url, site) {
 
     const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
 
-    const rawTags = data.keywords.split(', ');
+    const poster = videoData.picPreview;
+    const trailer = `${videoData.playerOptions.host}${videoData.url}`;
+
+    const photoPath = $('.picturesItem a').attr('href');
+    const photos = await getPhotos(photoPath, site);
+
+    const tags = data.keywords.split(', ');
     const siteName = data.productionCompany ? data.productionCompany.name : $('#logoLink a').attr('title');
-    const siteId = siteName && siteName.replace(/\s+/g, '').toLowerCase();
+    const channel = siteName && siteName.replace(/\s+/g, '').toLowerCase();
-
-    const [channelSite, tags] = await Promise.all([
-        site.isFallback
-            ? knex('sites')
-                .where({ slug: siteId })
-                .orWhereRaw('name = ? collate NOCASE', [siteName])
-                .first()
-            : site,
-        matchTags(rawTags),
-    ]);
-
-    // only replace generic URL with site URL if site is not marked to fetch scenes from generic site
-    const originalUrl = channelSite && !(channelSite.parameters && JSON.parse(channelSite.parameters).filter)
-        ? `${channelSite.url}/en/video/${new URL(url).pathname.split('/').slice(-2).join('/')}`
-        : url;
 
     return {
-        url: originalUrl,
+        url,
         entryId,
         title,
         date,
@@ -112,22 +147,30 @@ async function scrapeScene(html, url, site) {
         description,
         duration,
         tags,
+        poster,
+        photos,
+        trailer: {
+            src: trailer,
+        },
         rating: {
             likes,
             dislikes,
         },
-        site: channelSite || site,
+        site,
+        channel,
     };
 }
 
 async function fetchLatest(site, page = 1) {
-    const res = await bhttp.get(`${site.parameters && site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`);
+    const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/latest/${page}`;
+    const res = await bhttp.get(url);
 
     return scrape(res.body.toString(), site);
 }
 
 async function fetchUpcoming(site) {
-    const res = await bhttp.get(`${site.parameters && site.parameters.filter ? 'https://21sextury.com' : site.url}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`);
+    const url = `${site.url || 'https://21sextury.com'}/en/videos/All-Categories/0/All-Pornstars/0/upcoming`;
+    const res = await bhttp.get(url);
 
     return scrape(res.body.toString(), site);
 }

src/scrapers/ddfnetwork.js

@@ -6,7 +6,6 @@ const { JSDOM } = require('jsdom');
 const moment = require('moment');
 
 const knex = require('../knex');
-const { matchTags } = require('../tags');
 
 /* eslint-disable newline-per-chained-call */
 function scrapeLatest(html, site) {
@@ -49,13 +48,16 @@ async function scrapeScene(html, url, site) {
     const title = $('meta[itemprop="name"]').attr('content');
     const description = $('.descr-box p').text(); // meta tags don't contain full description
 
-    const date = moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate();
+    const dateProp = $('meta[itemprop="uploadDate"]').attr('content');
+    const date = dateProp
+        ? moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate()
+        : moment.utc($('.title-border:nth-child(2) p').text(), 'MM.DD.YYYY').toDate();
     const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
 
     const likes = Number($('.info-panel.likes .likes').text());
     const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
 
-    const rawTags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
+    const tags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
 
     const poster = $('#video').attr('poster');
     const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
@@ -63,21 +65,7 @@ async function scrapeScene(html, url, site) {
     const trailer540 = $('source[res="540"]').attr('src');
     const trailer720 = $('source[res="720"]').attr('src');
 
-    /*
-     * broken as of nov 2019
-    const { origin } = new URL($('.pornstar-card meta[itemprop="url"]').first().attr('content'));
-
-    const [channelSite, tags] = await Promise.all([
-        // don't find site if original is already specific
-        site.isFallback ? knex('sites').where({ url: origin }).first() : site,
-        matchTags(rawTags),
-    ]);
-    */
-
-    const tags = await matchTags(rawTags);
-
     return {
-        // url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
         url,
         entryId,
         title,
@@ -88,20 +76,19 @@ async function scrapeScene(html, url, site) {
         tags,
         poster,
         photos,
-        trailer: trailer540
+        trailer: [
-            ? {
+            {
-                src: trailer540,
-                quality: 540,
-            }
-            : {
-                // backup
                 src: trailer720,
                 quality: 720,
             },
+            {
+                src: trailer540,
+                quality: 540,
+            },
+        ],
         rating: {
             likes,
         },
-        // site: channelSite || site,
         site,
     };
 }

src/scrapers/dogfart.js

@@ -8,7 +8,6 @@ const moment = require('moment');
 
 const knex = require('../knex');
 const { matchTags } = require('../tags');
-const pluckPhotos = require('../utils/pluck-photos');
 
 async function getPhoto(url) {
     const res = await bhttp.get(url);
@@ -20,7 +19,7 @@ async function getPhoto(url) {
     return photoUrl;
 }
 
-async function getPhotos(albumUrl, site, siteUrl) {
+async function getPhotos(albumUrl) {
     const res = await bhttp.get(albumUrl);
     const html = res.body.toString();
     const { document } = new JSDOM(html).window;
@@ -28,15 +27,7 @@ async function getPhotos(albumUrl, site, siteUrl) {
     const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
     const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
 
-    // dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween
+    const photoUrls = await Promise.map(Array.from({ length: lastPhotoIndex }), async (index) => {
-    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
-    const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);
-
-    if (photoLimit > 25) {
-        console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
-    }
-
-    const photoUrls = await Promise.map(photoIndexes, async (index) => {
         const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`;
 
         return getPhoto(pageUrl);

src/scrapers/julesjordan.js

@@ -9,8 +9,6 @@ const moment = require('moment');
 const { heightToCm } = require('../utils/convert');
 const { matchTags } = require('../tags');
 
-const pluckPhotos = require('../utils/pluck-photos');
-
 async function fetchPhotos(url) {
     const res = await bhttp.get(url);
 
@@ -58,14 +56,7 @@ async function getPhotos(entryId, site, page = 1) {
         })
         : [];
 
-    const allPhotos = photos.concat(otherPhotos.flat());
+    return photos.concat(otherPhotos.flat());
-
-    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
-    const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
-
-    const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
-
-    return pluckedPhotos;
 }
 
 function scrapeLatest(html, site) {

src/scrapers/xempire.js

@@ -8,7 +8,6 @@ const moment = require('moment');
 
 const { fetchSites } = require('../sites');
 const { matchTags } = require('../tags');
-const pluckPhotos = require('../utils/pluck-photos');
 
 const defaultTags = {
     hardx: [],
@@ -38,7 +37,7 @@ function scrapePhotos(html) {
     return unlockedPhotos.concat(lockedThumbnails);
 }
 
-async function getPhotos(albumPath, siteDomain, site) {
+async function getPhotos(albumPath, siteDomain) {
     const albumUrl = `https://${siteDomain}${albumPath}`;
 
     const html = await fetchPhotos(albumUrl);
@@ -56,14 +55,7 @@ async function getPhotos(albumPath, siteDomain, site) {
         concurrency: 2,
     });
 
-    const allPhotos = photos.concat(otherPhotos.flat());
+    return photos.concat(otherPhotos.flat());
-
-    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
-    const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
-
-    const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
-
-    return pluckedPhotos;
 }
 
 function scrape(html, site) {

src/utils/pluck-photos.js

@@ -1,13 +1,18 @@
 'use strict';
 
+const config = require('config');
+
 // pick {photoLimit} photos evenly distributed photos from a set with {photoTotal} photos, return array of indexes starting at 1
-function pluckPhotos(photoTotal, photoLimit) {
+function pluckPhotos(photos, release, specifiedLimit) {
+    const limit = specifiedLimit || config.media.limit;
+    console.log(limit);
+
     const plucked = [1]
         .concat(
-            Array.from({ length: photoLimit - 1 }, (value, index) => Math.round((index + 1) * (photoTotal / (photoLimit - 1)))),
+            Array.from({ length: limit - 1 }, (value, index) => Math.round((index + 1) * (photos.length / (limit)))),
         );
 
-    return Array.from(new Set(plucked)); // remove duplicates, may happen when photo total and photo limit are close
+    return Array.from(new Set(plucked)).map(photoIndex => photos[photoIndex]); // remove duplicates, may happen when photo total and photo limit are close
 }
 
 module.exports = pluckPhotos;

traxxx

@@ -1,2 +1,2 @@
 #!/usr/bin/bash
-node ./src/app.js "$@";
+node ./dist/init.js "$@";