Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

assets/components/actor/actor.vue

@@ -6,20 +6,16 @@
         <FilterBar :fetch-releases="fetchReleases" />
 
         <div class="header">
-            <h2 class="title">{{ actor.name }}</h2>
-        </div>
+            <img
+                v-if="actor.avatars && actor.avatars.length > 0"
+                :src="`/media/${actor.avatars[0].path}`"
+                class="avatar"
+            >
+
+            <ul class="bio sidebar nolist">
+                <h2 class="title">{{ actor.name }}</h2>
 
-        <div class="content-inner">
-            <div class="avatars">
-                <img
-                    v-for="avatar in actor.avatars"
-                    :key="`avatar-${avatar.id}`"
-                    :src="`/media/${avatar.path}`"
-                    class="avatar"
-                >
-            </div>
 
-            <ul class="bio">
                 <li v-if="actor.aliases.length">
                     <dfn class="bio-heading">Also known as</dfn>
                     <span>{{ actor.aliases.join(', ') }}</span>
@@ -78,12 +74,31 @@
 
                 <li v-if="actor.social && actor.social.length > 0">
                     <dfn class="bio-heading">Social</dfn>
-                    {{ actor.social.join(',') }}
+                    <a
+                        v-for="social in actor.social"
+                        :key="`social-${social.id}`"
+                        :href="social.url"
+                        target="_blank"
+                        rel="noopener noreferrer"
+                        class="social"
+                    >{{ social.platform || social.url }}</a>
                 </li>
+
+                <li class="description">{{ actor.description }}</li>
+
+                <div class="photos">
+                    <img
+                        v-for="avatar in actor.avatars.slice(1)"
+                        :key="`avatar-${avatar.id}`"
+                        :src="`/media/${avatar.path}`"
+                        class="photo"
+                    >
+                </div>
             </ul>
 
-            <span class="description">{{ actor.description }}</span>
+        </div>
 
+        <div class="content-inner">
             <Releases
                 :releases="releases"
                 :context="actor.name"
@@ -170,13 +185,18 @@ export default {
     padding: 1rem;
 }
 
-.avatars {
-    padding: 1rem;
-}
-
 .avatar {
     height: 20rem;
     margin: 0 1rem 0 0;
+    display: block;
+}
+
+.photo {
+    height: 10rem;
+}
+
+.social {
+    display: block;
 }
 
 .flag {
@@ -184,4 +204,8 @@ export default {
     border: solid 1px $shadow-weak;
     margin: 0 .25rem 0 0;
 }
+
+.releases {
+    flex-grow: 1;
+}
 </style>

migrations/20190325001339_releases.js

@@ -226,6 +226,20 @@ exports.up = knex => Promise.resolve()
         table.datetime('created_at')
             .defaultTo(knex.fn.now());
     }))
+    .then(() => knex.schema.createTable('social', (table) => {
+        table.increments('id', 16);
+
+        table.string('url');
+        table.string('platform');
+
+        table.string('domain');
+        table.integer('target_id', 16);
+
+        table.unique(['url', 'domain', 'target_id']);
+
+        table.datetime('created_at')
+            .defaultTo(knex.fn.now());
+    }))
     .then(() => knex.schema.createTable('actors_associated', (table) => {
         table.increments('id', 16);
 
@@ -281,6 +295,7 @@ exports.down = knex => Promise.resolve()
     .then(() => knex.schema.dropTable('tags'))
     .then(() => knex.schema.dropTable('tags_groups'))
     .then(() => knex.schema.dropTable('media'))
+    .then(() => knex.schema.dropTable('social'))
     .then(() => knex.schema.dropTable('actors'))
     .then(() => knex.schema.dropTable('releases'))
     .then(() => knex.schema.dropTable('sites'))

public/css/style.css

@@ -599,18 +599,25 @@
 .description[data-v-677a8360] {
   padding: 1rem;
 }
-.avatars[data-v-677a8360] {
-  padding: 1rem;
-}
 .avatar[data-v-677a8360] {
   height: 20rem;
   margin: 0 1rem 0 0;
+  display: block;
+}
+.photo[data-v-677a8360] {
+  height: 10rem;
+}
+.social[data-v-677a8360] {
+  display: block;
 }
 .flag[data-v-677a8360] {
   height: 1rem;
   border: solid 1px rgba(0, 0, 0, 0.2);
   margin: 0 .25rem 0 0;
 }
+.releases[data-v-677a8360] {
+  flex-grow: 1;
+}
 
 /* $primary: #ff886c; */
 .header[data-v-80991bcc] {

seeds/00_networks.js

@@ -2,120 +2,120 @@
 const upsert = require('../src/utils/upsert');
 
 const networks = [
-        {
-            slug: '21sextury',
-            name: '21Sextury',
-            url: 'https://www.21sextury.com',
-            description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
-        },
-        {
-            slug: 'bangbros',
-            name: 'Bang Bros',
-            url: 'https://bangbros.com',
-            description: 'Here at Bang Bros, we only film the best highest quality porn with the sexiest Amateur girls and the top pornstars. Updated daily on Bangbros.com.',
-        },
-        {
-            slug: 'blowpass',
-            name: 'Blowpass',
-            url: 'https://www.blowpass.com',
-            description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
-        },
-        {
-            slug: 'brazzers',
-            name: 'Brazzers',
-            url: 'https://www.brazzers.com',
-            description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
-        },
-        {
-            slug: 'ddfnetwork',
-            name: 'DDF Network',
-            url: 'https://ddfnetwork.com',
-            description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
-        },
-        {
-            slug: 'dogfartnetwork',
-            name: 'Dogfart Network',
-            url: 'https://dogfartnetwork.com',
-            description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.',
-            parameters: JSON.stringify({ photoLimit: 25 }),
-        },
-        {
-            slug: 'evilangel',
-            name: 'Evil Angel',
-            url: 'https://evilangel.com',
-            description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
-        },
-        {
-            slug: 'julesjordan',
-            name: 'Jules Jordan',
-            url: 'https://www.julesjordan.com',
-        },
-        {
-            slug: 'kink',
-            name: 'Kink',
-            url: 'https://www.kink.com',
-            description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
-        },
-        {
-            slug: 'legalporno',
-            name: 'LegalPorno',
-            url: 'https://www.legalporno.com',
-            description: 'The Best HD Porn For You!',
-        },
-        {
-            slug: 'mikeadriano',
-            name: 'Mike Adriano',
-            url: null,
-            description: null,
-        },
-        {
-            slug: 'mofos',
-            name: 'MOFOS',
-            url: 'https://www.mofos.com',
-            description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
-        },
-        {
-            slug: 'naughtyamerica',
-            name: 'Naughty America',
-            url: 'https://www.naughtyamerica.com',
-            description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!',
-        },
-        {
-            slug: 'pervcity',
-            name: 'Perv City',
-            url: 'https://www.pervcity.com',
-            description: '',
-        },
-        {
-            slug: 'pornpros',
-            name: 'Porn Pros',
-            url: 'https://pornpros.com',
-            description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.',
-        },
-        {
-            slug: 'private',
-            name: 'Private',
-            url: 'https://www.private.com',
-            description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
-        },
-        {
-            slug: 'realitykings',
-            name: 'Reality Kings',
-            url: 'https://www.realitykings.com',
-            description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
-        },
-        {
-            slug: 'vixen',
-            name: 'Vixen',
-            url: 'https://www.vixen.com',
-            description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
-        },
-        {
-            slug: 'xempire',
-            name: 'XEmpire',
-            url: 'https://www.xempire.com',
-            description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
-        },
+    {
+        slug: '21sextury',
+        name: '21Sextury',
+        url: 'https://www.21sextury.com',
+        description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
+    },
+    {
+        slug: 'bangbros',
+        name: 'Bang Bros',
+        url: 'https://bangbros.com',
+        description: 'Here at Bang Bros, we only film the best highest quality porn with the sexiest Amateur girls and the top pornstars. Updated daily on Bangbros.com.',
+    },
+    {
+        slug: 'blowpass',
+        name: 'Blowpass',
+        url: 'https://www.blowpass.com',
+        description: 'Welcome to Blowpass.com, your ultimate source for deepthroat porn, MILF and teen blowjob videos, big cumshots and any and everything oral!',
+    },
+    {
+        slug: 'brazzers',
+        name: 'Brazzers',
+        url: 'https://www.brazzers.com',
+        description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
+    },
+    {
+        slug: 'ddfnetwork',
+        name: 'DDF Network',
+        url: 'https://ddfnetwork.com',
+        description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
+    },
+    {
+        slug: 'dogfartnetwork',
+        name: 'Dogfart Network',
+        url: 'https://dogfartnetwork.com',
+        description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.',
+        parameters: JSON.stringify({ photoLimit: 25 }),
+    },
+    {
+        slug: 'evilangel',
+        name: 'Evil Angel',
+        url: 'https://evilangel.com',
+        description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.',
+    },
+    {
+        slug: 'julesjordan',
+        name: 'Jules Jordan',
+        url: 'https://www.julesjordan.com',
+    },
+    {
+        slug: 'kink',
+        name: 'Kink',
+        url: 'https://www.kink.com',
+        description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
+    },
+    {
+        slug: 'legalporno',
+        name: 'LegalPorno',
+        url: 'https://www.legalporno.com',
+        description: 'The Best HD Porn For You!',
+    },
+    {
+        slug: 'mikeadriano',
+        name: 'Mike Adriano',
+        url: null,
+        description: null,
+    },
+    {
+        slug: 'mofos',
+        name: 'MOFOS',
+        url: 'https://www.mofos.com',
+        description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
+    },
+    {
+        slug: 'naughtyamerica',
+        name: 'Naughty America',
+        url: 'https://www.naughtyamerica.com',
+        description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!',
+    },
+    {
+        slug: 'pervcity',
+        name: 'Perv City',
+        url: 'https://www.pervcity.com',
+        description: '',
+    },
+    {
+        slug: 'pornpros',
+        name: 'Porn Pros',
+        url: 'https://pornpros.com',
+        description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.',
+    },
+    {
+        slug: 'private',
+        name: 'Private',
+        url: 'https://www.private.com',
+        description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
+    },
+    {
+        slug: 'realitykings',
+        name: 'Reality Kings',
+        url: 'https://www.realitykings.com',
+        description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
+    },
+    {
+        slug: 'vixen',
+        name: 'Vixen',
+        url: 'https://www.vixen.com',
+        description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
+    },
+    {
+        slug: 'xempire',
+        name: 'XEmpire',
+        url: 'https://www.xempire.com',
+        description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
+    },
 ];
 
 exports.seed = knex => Promise.resolve()

seeds/02_tags.js

@@ -1,5 +1,3 @@
-'use strict';
-
 const upsert = require('../src/utils/upsert');
 
 const groups = [
@@ -61,7 +59,7 @@ function getTags(groupsMap) {
             name: '69',
             slug: '69',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'airtight',
@@ -69,7 +67,7 @@ function getTags(groupsMap) {
             alias_for: null,
             description: 'A cock in every penetrable hole (of a woman); one in the mouth, one in the vagina, and one in the asshole.',
             priority: 9,
-            group_id: groupsMap['penetration'],
+            group_id: groupsMap.penetration,
         },
         {
             name: 'amateur',
@@ -80,13 +78,13 @@ function getTags(groupsMap) {
             name: 'american',
             slug: 'american',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'anal creampie',
             slug: 'anal-creampie',
             alias_for: null,
-            description: 'Ejaculating into the asshole.'
+            description: 'Ejaculating into the asshole.',
         },
         {
             name: 'anal',
@@ -120,13 +118,13 @@ function getTags(groupsMap) {
             name: 'asian',
             slug: 'asian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'athletic',
             slug: 'athletic',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'ass to mouth',
@@ -148,13 +146,13 @@ function getTags(groupsMap) {
             name: 'ballerina',
             slug: 'ballerina',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'bathroom',
             slug: 'bathroom',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
         {
             name: 'BDSM',
@@ -165,25 +163,25 @@ function getTags(groupsMap) {
             name: 'BBC',
             slug: 'bbc',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big cock',
             slug: 'big-cock',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big butt',
             slug: 'big-butt',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big boobs',
             slug: 'big-boobs',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'bisexual',
@@ -194,13 +192,13 @@ function getTags(groupsMap) {
             name: 'black hair',
             slug: 'black-hair',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'blonde',
             slug: 'blonde',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'blowjob',
@@ -211,7 +209,7 @@ function getTags(groupsMap) {
             name: 'blowbang',
             slug: 'blowbang',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'bondage',
@@ -222,7 +220,7 @@ function getTags(groupsMap) {
             name: 'brunette',
             slug: 'brunette',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'bukkake',
@@ -233,7 +231,7 @@ function getTags(groupsMap) {
             name: 'cheerleader',
             slug: 'cheerleader',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'choking',
@@ -325,13 +323,13 @@ function getTags(groupsMap) {
             name: 'dress',
             slug: 'dress',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'ebony',
             slug: 'ebony',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'electric shock',
@@ -347,19 +345,19 @@ function getTags(groupsMap) {
             name: 'European',
             slug: 'european',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'facefuck',
             slug: 'facefuck',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'facesitting',
             slug: 'facesitting',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'facial',
@@ -390,7 +388,7 @@ function getTags(groupsMap) {
             name: 'FMF threesome',
             slug: 'fmf',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'gag',
@@ -402,7 +400,7 @@ function getTags(groupsMap) {
             slug: 'gangbang',
             alias_for: null,
             priority: 9,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'gapes',
@@ -424,7 +422,7 @@ function getTags(groupsMap) {
             name: 'hairy',
             slug: 'hairy',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'hardcore',
@@ -435,13 +433,13 @@ function getTags(groupsMap) {
             name: 'high heels',
             slug: 'high-heels',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'hungarian',
             slug: 'hungarian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'humiliation',
@@ -493,13 +491,13 @@ function getTags(groupsMap) {
             name: 'lingerie',
             slug: 'lingerie',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'maid',
             slug: 'maid',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'masturbation',
@@ -510,31 +508,31 @@ function getTags(groupsMap) {
             name: 'MILF',
             slug: 'milf',
             alias_for: null,
-            group_id: groupsMap['age'],
+            group_id: groupsMap.age,
         },
         {
             name: 'MFM threesome',
             slug: 'mfm',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'miniskirt',
             slug: 'miniskirt',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'missionary',
             slug: 'missionary',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'natural boobs',
             slug: 'natural-boobs',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'nipple clamps',
@@ -550,13 +548,13 @@ function getTags(groupsMap) {
             name: 'orgy',
             slug: 'orgy',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'outdoors',
             slug: 'outdoors',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
         {
             name: 'outie pussy',
@@ -597,7 +595,7 @@ function getTags(groupsMap) {
             name: 'redhead',
             slug: 'redhead',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'reverse cowgirl',
@@ -618,7 +616,7 @@ function getTags(groupsMap) {
             name: 'russian',
             slug: 'russian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'saliva',
@@ -629,7 +627,7 @@ function getTags(groupsMap) {
             name: 'schoolgirl',
             slug: 'schoolgirl',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'shaved',
@@ -640,19 +638,19 @@ function getTags(groupsMap) {
             name: 'shoes on',
             slug: 'shoes-on',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'short hair',
             slug: 'short-hair',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'skirt',
             slug: 'skirt',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'slapping',
@@ -668,7 +666,7 @@ function getTags(groupsMap) {
             name: 'socks',
             slug: 'socks',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'spanking',
@@ -679,7 +677,7 @@ function getTags(groupsMap) {
             name: 'spooning',
             slug: 'spooning',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'strapon',
@@ -715,7 +713,7 @@ function getTags(groupsMap) {
             name: 'stockings',
             slug: 'stockings',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'strap-on dildo',
@@ -736,19 +734,19 @@ function getTags(groupsMap) {
             name: 'tattoo',
             slug: 'tattoo',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'threesome',
             slug: 'threesome',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'teen',
             slug: 'teen',
             alias_for: null,
-            group_id: groupsMap['age'],
+            group_id: groupsMap.age,
         },
         {
             name: 'titty fuck',
@@ -800,7 +798,7 @@ function getTags(groupsMap) {
             name: 'white',
             slug: 'white',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'wife',
@@ -811,7 +809,7 @@ function getTags(groupsMap) {
             name: 'office',
             slug: 'office',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
     ];
 }
@@ -820,23 +818,23 @@ function getTagAliases(tagsMap) {
     return [
         {
             name: '2-on-1',
-            alias_for: tagsMap['threesome'],
+            alias_for: tagsMap.threesome,
         },
         {
             name: '2 on 1',
-            alias_for: tagsMap['threesome'],
+            alias_for: tagsMap.threesome,
         },
         {
             name: '3+ on 1',
-            alias_for: tagsMap['gangbang'],
+            alias_for: tagsMap.gangbang,
         },
         {
             name: 'anal sex',
-            alias_for: tagsMap['anal'],
+            alias_for: tagsMap.anal,
         },
         {
             name: 'anal gape',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'anilingus',
@@ -844,7 +842,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'asians',
-            alias_for: tagsMap['asian'],
+            alias_for: tagsMap.asian,
         },
         {
             name: 'anal fingering',
@@ -856,7 +854,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'ass fucking',
-            alias_for: tagsMap['anal'],
+            alias_for: tagsMap.anal,
         },
         {
             name: 'atm',
@@ -864,11 +862,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'bald pussy',
-            alias_for: tagsMap['shaved'],
+            alias_for: tagsMap.shaved,
         },
         {
             name: 'ball gag',
-            alias_for: tagsMap['gag'],
+            alias_for: tagsMap.gag,
         },
         {
             name: 'boob fucking',
@@ -876,19 +874,19 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'mfm',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'fmf',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'ffm',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'bgb',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'big ass',
@@ -896,11 +894,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'big black cock',
-            alias_for: tagsMap['bbc'],
+            alias_for: tagsMap.bbc,
         },
         {
             name: 'big black cocks',
-            alias_for: tagsMap['bbc'],
+            alias_for: tagsMap.bbc,
         },
         {
             name: 'big cocks',
@@ -924,39 +922,47 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'bi',
-            alias_for: tagsMap['bisexual'],
+            alias_for: tagsMap.bisexual,
         },
         {
             name: 'black',
-            alias_for: tagsMap['ebony'],
+            alias_for: tagsMap.ebony,
         },
         {
             name: 'blonde hair',
-            alias_for: tagsMap['blonde'],
+            alias_for: tagsMap.blonde,
         },
         {
             name: 'blondes',
-            alias_for: tagsMap['blonde'],
+            alias_for: tagsMap.blonde,
         },
         {
             name: 'blow job',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjobs',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjob pov',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjob (double)',
             alias_for: tagsMap['double-blowjob'],
         },
+        {
+            name: 'blowjob - double',
+            alias_for: tagsMap['double-blowjob'],
+        },
         {
             name: 'blowjob (pov)',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
+        },
+        {
+            name: 'blowjob - pov',
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'boob job',
@@ -968,11 +974,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'brown hair',
-            alias_for: tagsMap['brunette'],
+            alias_for: tagsMap.brunette,
         },
         {
             name: 'brunettes',
-            alias_for: tagsMap['brunette'],
+            alias_for: tagsMap.brunette,
         },
         {
             name: 'buttplug',
@@ -992,7 +998,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'cheer leader',
-            alias_for: tagsMap['cheerleader'],
+            alias_for: tagsMap.cheerleader,
         },
         {
             name: 'clover clamps',
@@ -1000,11 +1006,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'couples fantasies',
-            alias_for: tagsMap['couples'],
+            alias_for: tagsMap.couples,
         },
         {
             name: 'creampies',
-            alias_for: tagsMap['creampie'],
+            alias_for: tagsMap.creampie,
         },
         {
             name: 'crop', // a type of whip, not [sic] short for corporal
@@ -1028,11 +1034,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'cum swallowing',
-            alias_for: tagsMap['swallowing'],
+            alias_for: tagsMap.swallowing,
         },
         {
             name: 'cum shot',
-            alias_for: tagsMap['cumshot'],
+            alias_for: tagsMap.cumshot,
         },
         {
             name: 'cunnilingus',
@@ -1044,15 +1050,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'deep throat',
-            alias_for: tagsMap['deepthroat'],
+            alias_for: tagsMap.deepthroat,
         },
         {
             name: 'deepthroating',
-            alias_for: tagsMap['deepthroat'],
+            alias_for: tagsMap.deepthroat,
         },
         {
             name: 'dildo',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'doggystyle',
@@ -1072,15 +1078,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'dom',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'domination',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'dominatrix',
-            alias_for: tagsMap['femdom'],
+            alias_for: tagsMap.femdom,
         },
         {
             name: 'dp',
@@ -1090,6 +1096,10 @@ function getTagAliases(tagsMap) {
             name: 'double penetration (dp)',
             alias_for: tagsMap['double-penetration'],
         },
+        {
+            name: 'double penetration - dp',
+            alias_for: tagsMap['double-penetration'],
+        },
         {
             name: 'dap',
             alias_for: tagsMap['double-anal'],
@@ -1132,7 +1142,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'drool',
-            alias_for: tagsMap['saliva'],
+            alias_for: tagsMap.saliva,
         },
         {
             name: 'enhanced',
@@ -1144,27 +1154,27 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'facefucking',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face fuck',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face fucking',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face sitting',
-            alias_for: tagsMap['facesitting'],
+            alias_for: tagsMap.facesitting,
         },
         {
             name: 'facial cumshot',
-            alias_for: tagsMap['facial'],
+            alias_for: tagsMap.facial,
         },
         {
             name: 'facials',
-            alias_for: tagsMap['facial'],
+            alias_for: tagsMap.facial,
         },
         {
             name: 'fake boobs',
@@ -1180,27 +1190,27 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'foot fetish',
-            alias_for: tagsMap['feet'],
+            alias_for: tagsMap.feet,
         },
         {
             name: 'french kissing',
-            alias_for: tagsMap['kissing'],
+            alias_for: tagsMap.kissing,
         },
         {
             name: 'gape',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'gaping',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'gapes (gaping asshole)',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'group sex',
-            alias_for: tagsMap['orgy'],
+            alias_for: tagsMap.orgy,
         },
         {
             name: 'flagellation',
@@ -1212,7 +1222,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'huge toys',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'innie',
@@ -1224,23 +1234,23 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'lezdom',
-            alias_for: tagsMap['lesbian'],
+            alias_for: tagsMap.lesbian,
         },
         {
             name: 'mini-skirt',
-            alias_for: tagsMap['miniskirt'],
+            alias_for: tagsMap.miniskirt,
         },
         {
             name: 'mmf',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'mff',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'mature & milf',
-            alias_for: tagsMap['milf'],
+            alias_for: tagsMap.milf,
         },
         {
             name: 'natural',
@@ -1252,7 +1262,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'oral',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'outie',
@@ -1260,11 +1270,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'piercing',
-            alias_for: tagsMap['piercings'],
+            alias_for: tagsMap.piercings,
         },
         {
             name: 'pierced',
-            alias_for: tagsMap['piercings'],
+            alias_for: tagsMap.piercings,
         },
         {
             name: 'prolapse',
@@ -1284,11 +1294,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'red hair',
-            alias_for: tagsMap['redhead'],
+            alias_for: tagsMap.redhead,
         },
         {
             name: 'red head',
-            alias_for: tagsMap['redhead'],
+            alias_for: tagsMap.redhead,
         },
         {
             name: 'rimming',
@@ -1300,39 +1310,39 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'role play',
-            alias_for: tagsMap['roleplay'],
+            alias_for: tagsMap.roleplay,
         },
         {
             name: 'rope bondage',
-            alias_for: tagsMap['bondage'],
+            alias_for: tagsMap.bondage,
         },
         {
             name: 'rough sex',
-            alias_for: tagsMap['rough'],
+            alias_for: tagsMap.rough,
         },
         {
             name: 'school girl',
-            alias_for: tagsMap['schoolgirl'],
+            alias_for: tagsMap.schoolgirl,
         },
         {
             name: 'sadomasochism',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'sadism',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'scissoring',
-            alias_for: tagsMap['lesbian'],
+            alias_for: tagsMap.lesbian,
         },
         {
             name: 'sex toys',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'shaved pussy',
-            alias_for: tagsMap['shaved'],
+            alias_for: tagsMap.shaved,
         },
         {
             name: 'shoes',
@@ -1340,7 +1350,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'slave',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'small ass',
@@ -1352,11 +1362,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'spit',
-            alias_for: tagsMap['saliva'],
+            alias_for: tagsMap.saliva,
         },
         {
             name: 'spitroast',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'standing doggystyle',
@@ -1364,7 +1374,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'swallow',
-            alias_for: tagsMap['swallowing'],
+            alias_for: tagsMap.swallowing,
         },
         {
             name: 'strap-on',
@@ -1380,19 +1390,19 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'sub',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'submission',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'tattoos',
-            alias_for: tagsMap['tattoo'],
+            alias_for: tagsMap.tattoo,
         },
         {
             name: 'teens',
-            alias_for: tagsMap['teen'],
+            alias_for: tagsMap.teen,
         },
         {
             name: 'tiny boobs',
@@ -1412,15 +1422,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'trans',
-            alias_for: tagsMap['transsexual'],
+            alias_for: tagsMap.transsexual,
         },
         {
             name: 'trimmed pussy',
-            alias_for: tagsMap['trimmed'],
+            alias_for: tagsMap.trimmed,
         },
         {
             name: 'ts',
-            alias_for: tagsMap['transsexual'],
+            alias_for: tagsMap.transsexual,
         },
         {
             name: 'whipping',
@@ -1428,11 +1438,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'work',
-            alias_for: tagsMap['office'],
+            alias_for: tagsMap.office,
         },
         {
             name: 'workplace',
-            alias_for: tagsMap['office'],
+            alias_for: tagsMap.office,
         },
         {
             name: 'zapper',

src/actors.js

@@ -9,11 +9,13 @@ const whereOr = require('./utils/where-or');
 const { createActorMediaDirectory, storeAvatars } = require('./media');
 
 async function curateActor(actor) {
-    const [aliases, avatars] = await Promise.all([
+    const [aliases, avatars, social] = await Promise.all([
         knex('actors').where({ alias_for: actor.id }),
         knex('media')
             .where({ domain: 'actors', target_id: actor.id })
             .orderBy('index'),
+        knex('social')
+            .where({ domain: 'actors', target_id: actor.id }),
     ]);
 
     return {
@@ -46,6 +48,7 @@ async function curateActor(actor) {
         aliases: aliases.map(({ name }) => name),
         slug: actor.slug,
         avatars,
+        social,
     };
 }
 
@@ -94,6 +97,35 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
     return curatedActor;
 }
 
+function curateSocialEntry(url, actor) {
+    const { hostname, origin, pathname } = new URL(url);
+    const platform = ['twitter', 'instagram', 'snapchat', 'modelhub', 'youtube'].find(platformName => hostname.match(platformName));
+
+    return {
+        url: `${origin}${pathname}`,
+        platform,
+        domain: 'actors',
+        target_id: actor.id,
+    };
+}
+
+function curateSocialEntries(urls, actor) {
+    if (!urls) {
+        return [];
+    }
+
+    return urls.reduce((acc, url) => {
+        const socialEntry = curateSocialEntry(url, actor);
+
+        if (acc.some(entry => socialEntry.url === entry.url)) {
+            // prevent duplicates
+            return acc;
+        }
+
+        return [...acc, socialEntry];
+    }, []);
+}
+
 async function fetchActors(queryObject) {
     const releases = await knex('actors')
         .select(
@@ -112,34 +144,30 @@ async function fetchActors(queryObject) {
 async function storeActor(actor, scraped = false, scrapeSuccess = false) {
     const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
 
-    const actorEntries = await knex('actors')
+    const [actorEntry] = await knex('actors')
         .insert(curatedActor)
         .returning('*');
 
-    if (actorEntries.length) {
-        const actorEntry = actorEntries[0];
+    await knex('social').insert(curateSocialEntries(actor.social, actor));
 
-        console.log(`Added new entry for actor '${actor.name}'`);
+    console.log(`Added new entry for actor '${actor.name}'`);
 
-        return actorEntry;
-    }
-
-    console.error(`Unable to save profile for '${actor.name}'`);
-
-    return null;
+    return actorEntry;
 }
 
 async function updateActor(actor, scraped = false, scrapeSuccess = false) {
     const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
 
-    const actorEntries = await knex('actors')
+    const [actorEntry] = await knex('actors')
         .where({ id: actor.id })
         .update(curatedActor)
         .returning('*');
 
+    await knex('social').insert(curateSocialEntries(actor.social, actor));
+
     console.log(`Updated entry for actor '${actor.name}'`);
 
-    return actorEntries[0];
+    return actorEntry;
 }
 
 function mergeProfiles(profiles, actor) {
@@ -177,7 +205,6 @@ function mergeProfiles(profiles, actor) {
     }, {
         social: [],
         avatars: [],
-        ...actor,
     });
 }
 

src/scrapers/brazzers.js

@@ -148,7 +148,7 @@ function scrapeProfile(html, url, actorName) {
     };
 
     if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
-    if (bio.Measurements && bio.Measurements.match(/\w+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
+    if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
     if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
     if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
     if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();

src/scrapers/mofos.js

@@ -1,67 +1,84 @@
 'use strict';
 
+const Promise = require('bluebird');
 const bhttp = require('bhttp');
-const cheerio = require('cheerio');
+const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
 const moment = require('moment');
 
-const knex = require('../knex');
+const { fetchSites } = require('../sites');
+const { cookieToData } = require('../utils/cookies');
 const { matchTags } = require('../tags');
 
+function getThumbs(scene) {
+    if (scene.images.poster) {
+        return scene.images.poster.map(image => image.xl.url);
+    }
+
+    if (scene.images.card_main_rect) {
+        return scene.images.card_main_rect
+            .concat(scene.images.card_secondary_rect || [])
+            .map(image => image.xl.url.replace('.thumb', ''));
+    }
+
+    return [];
+}
+
 /* eslint-disable newline-per-chained-call */
-function scrape(html, site) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElements = $('.widget-release-card').toArray();
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
+        const url = `https://www.mofos.com/scene/${entryId}/`;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
 
-    return sceneElements.map((element) => {
-        const sceneLinkElement = $(element).find('.title a');
+        const rawTags = data.tags.map(tag => tag.name);
+        const tags = await matchTags(rawTags);
 
-        const title = sceneLinkElement.text().trim();
-        const url = `https://www.mofos.com${sceneLinkElement.attr('href')}`;
-        const entryId = url.split('/').slice(-2, -1)[0];
-
-        const date = moment.utc($(element).find('.date-added').text(), 'MMM DD, YYYY').toDate();
-        const actors = $(element).find('.girls-name a').map((actorIndex, actorElement) => $(actorElement).attr('title').replace(/\s+/g, ' ')).toArray();
-
-        const stars = Number($(element).find('.rating').text().slice(0, -1).trim()) / 20;
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
         return {
             url,
             entryId,
             title,
+            description,
             actors,
-            date,
-            rating: {
-                stars,
+            tags,
+            poster,
+            photos,
+            trailer: {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
             },
+            date,
             site,
         };
-    });
+    }));
 }
 
-async function scrapeScene(html, url, site) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElement = $('.video-info');
+async function scrapeScene(data, url, site) {
+    const { id: entryId, title, description } = data;
+    const date = new Date(data.dateReleased);
+    const actors = data.actors.map(actor => actor.name);
 
-    const entryId = url.split('/').slice(-2, -1)[0];
-    const title = sceneElement.find('.title').text();
-    const description = sceneElement.find('.desc').text();
-    const actors = sceneElement.find('.girls-site-box a.model-name').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
+    const rawTags = data.tags.map(tag => tag.name);
 
-    const siteElement = sceneElement.find('.site-name');
-    const sitename = siteElement.text().trim();
-    const siteId = sitename.replace(/\s+/g, '').toLowerCase();
-    const siteUrl = siteElement.attr('href').split('/').slice(0, 4).join('/');
+    const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
-    const stars = Number(sceneElement.find('.rating-box .rating').text().slice(0, -1).trim()) / 20;
+    const siteName = data.collections[0].name;
+    const siteId = data.collections[0].id;
+    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
+    const siteUrl = `https://www.mofos.com/scenes?site=${siteId}`;
 
-    const rawTags = sceneElement.find('.categories a').map((tagIndex, tagElement) => $(tagElement).text().trim()).toArray();
-
-    const [channelSite, tags] = await Promise.all([
-        knex('sites')
-            .where({ slug: siteId })
-            .orWhere({ url: `https://www.mofos.com${siteUrl}` })
-            .orWhere({ name: sitename })
-            .first(),
+    const [[channelSite], tags] = await Promise.all([
+        site.isFallback
+            ? fetchSites({
+                slug: siteSlug,
+                name: siteName,
+                url: siteUrl,
+            })
+            : [site],
         matchTags(rawTags),
     ]);
 
@@ -72,35 +89,61 @@ async function scrapeScene(html, url, site) {
         description,
         actors,
         tags,
-        rating: {
-            stars,
+        poster,
+        photos,
+        trailer: {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
         },
-        site: channelSite || site,
+        date,
+        site: channelSite,
     };
 }
 
 async function fetchLatest(site, page = 1) {
-    const res = page > 1
-        ? await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/${page}/`)
-        : await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/`); // explicit page 1 redirects to homepage
+    const { search } = new URL(site.url);
+    const siteId = new URLSearchParams(search).get('site');
 
-    return scrape(res.body.toString(), site);
-}
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
 
-async function fetchUpcoming(site) {
-    const res = await bhttp.get(`${site.url}/all-models/all-categories/upcoming/bydate/`);
+    await session.get(site.url);
 
-    return scrape(res.body.toString(), site);
+    const cookieString = await cookieJar.getCookieStringAsync(site.url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
 }
 
 async function fetchScene(url, site) {
-    const res = await bhttp.get(url);
+    const entryId = url.match(/\d+/)[0];
 
-    return scrapeScene(res.body.toString(), url, site);
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeScene(res.body.result, url, site);
 }
 
 module.exports = {
     fetchLatest,
-    fetchUpcoming,
     fetchScene,
 };

src/scrapers/pornhub.js

@@ -59,7 +59,7 @@ async function scrapeProfile(html, _url, actorName) {
         profile.residenceCountry = residenceCountryEntry ? residenceCountryEntry.alpha2 : null;
     }
 
-    if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
+    if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-').map(measurement => parseInt(measurement, 10) || null);
     if (bio['Fake Boobs']) profile.naturalBoobs = bio['Fake Boobs'] === 'No';
 
     if (bio.Height) profile.height = Number(bio.Height.match(/\(\d+/)[0].slice(1));

src/scrapers/realitykings.js

@@ -4,8 +4,10 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
-const { JSDOM } = require('jsdom');
+const moment = require('moment');
 
+const { fetchSites } = require('../sites');
+const { cookieToData } = require('../utils/cookies');
 const { matchTags } = require('../tags');
 
 function getThumbs(scene) {
@@ -22,40 +24,18 @@ function getThumbs(scene) {
     return [];
 }
 
-async function scrapeLatest(html, site) {
-    const { document } = new JSDOM(html).window;
-
-    const scriptString = document.querySelector('script').textContent;
-    const prefix = 'window.__JUAN.initialState = {';
-
-    const dataStart = scriptString.slice(scriptString.indexOf(prefix) + prefix.length - 1);
-    const dataString = dataStart.slice(0, dataStart.indexOf('};') + 1);
-    const data = JSON.parse(dataString);
-
-    const actorsMap = data.entities.actors;
-    const tagsMap = data.entities.tags;
-    const scenes = Object.values(data.entities.releases);
-
-    return Promise.all(scenes.map(async (scene) => {
-        const {
-            id: entryId,
-            title,
-            description,
-        } = scene;
-
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
         const url = `https://www.realitykings.com/scene/${entryId}/`;
-        const date = new Date(scene.dateReleased);
-        const actors = scene.actors.map(actorId => actorsMap[actorId].name);
-        const duration = scene.videos.mediabook && scene.videos.mediabook.length;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
 
-        const rawTags = scene.tags.map(tagId => tagsMap[tagId].name);
+        const rawTags = data.tags.map(tag => tag.name);
         const tags = await matchTags(rawTags);
 
-        const [poster, ...photos] = getThumbs(scene);
-        const trailer720p = scene.videos.mediabook && scene.videos.mediabook.files['720p'] && scene.videos.mediabook.files['720p'].urls.view;
-        const trailer360p = scene.videos.mediabook && scene.videos.mediabook.files['360p'] && scene.videos.mediabook.files['360p'].urls.view;
-
-        const { likes, dislikes } = scene.stats;
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
         return {
             url,
@@ -63,41 +43,44 @@ async function scrapeLatest(html, site) {
             title,
             description,
             actors,
-            date,
             tags,
-            duration,
             poster,
             photos,
-            trailer: {
-                src: trailer720p || trailer360p,
-                quality: trailer720p ? 720 : 360,
+            trailer: trailer && {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
             },
-            rating: { likes, dislikes },
+            date,
             site,
         };
     }));
 }
 
 async function scrapeScene(data, url, site) {
-    const {
-        id: entryId,
-        title,
-        description,
-    } = data;
-
+    const { id: entryId, title, description } = data;
     const date = new Date(data.dateReleased);
     const actors = data.actors.map(actor => actor.name);
 
-    const { likes, dislikes } = data.stats;
-
     const rawTags = data.tags.map(tag => tag.name);
-    const tags = await matchTags(rawTags);
 
     const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
-    const duration = data.videos.mediabook && data.videos.mediabook.length;
-    const trailer720p = data.videos.mediabook && data.videos.mediabook.files['720p'] && data.videos.mediabook.files['720p'].urls.view;
-    const trailer360p = data.videos.mediabook && data.videos.mediabook.files['360p'] && data.videos.mediabook.files['360p'].urls.view;
+    const siteName = data.collections[0].name;
+    const siteId = data.collections[0].id;
+    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
+    const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
+
+    const [[channelSite], tags] = await Promise.all([
+        site.isFallback
+            ? fetchSites({
+                slug: siteSlug,
+                name: siteName,
+                url: siteUrl,
+            })
+            : [site],
+        matchTags(rawTags),
+    ]);
 
     return {
         url,
@@ -105,41 +88,56 @@ async function scrapeScene(data, url, site) {
         title,
         description,
         actors,
-        date,
-        duration,
         tags,
         poster,
         photos,
-        trailer: {
-            src: trailer720p || trailer360p,
-            quality: trailer720p ? 720 : 360,
+        trailer: trailer && {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
         },
-        rating: {
-            likes,
-            dislikes,
-        },
-        site,
+        date,
+        site: channelSite,
     };
 }
 
-async function fetchLatest(site, page = 1) {
+function getUrl(site) {
     const { hostname, search } = new URL(site.url);
 
     if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
-        const res = await bhttp.get(`${site.url}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return site.url;
     }
 
     if (site.parameters && site.parameters.siteId) {
-        const res = await bhttp.get(`https://www.realitykings.com/scenes?site=${site.parameters.siteId}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
     }
 
     throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
 }
 
+async function fetchLatest(site, page = 1) {
+    const url = getUrl(site);
+    const { search } = new URL(url);
+    const siteId = new URLSearchParams(search).get('site');
+
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
+}
+
 async function fetchScene(url, site) {
     const entryId = url.match(/\d+/)[0];
 
@@ -148,8 +146,8 @@ async function fetchScene(url, site) {
 
     await session.get(url);
 
-    const cookies = await cookieJar.getCookieStringAsync(url);
-    const instanceToken = cookies.split(';')[0].split('=')[1];
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
 
     const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
         headers: {
@@ -157,7 +155,7 @@ async function fetchScene(url, site) {
         },
     });
 
-    return scrapeScene(res.body.result.parent || res.body.result, url, site);
+    return scrapeScene(res.body.result, url, site);
 }
 
 module.exports = {

src/scrapers/xempire.js

@@ -7,6 +7,7 @@ const moment = require('moment');
 
 const knex = require('../knex');
 const { matchTags } = require('../tags');
+const pluckPhotos = require('../utils/pluck-photos');
 
 const defaultTags = {
     hardx: [],
@@ -36,7 +37,7 @@ function scrapePhotos(html) {
     return unlockedPhotos.concat(lockedThumbnails);
 }
 
-async function getPhotos(albumPath, siteDomain) {
+async function getPhotos(albumPath, siteDomain, site) {
     const albumUrl = `https://${siteDomain}${albumPath}`;
 
     const html = await fetchPhotos(albumUrl);
@@ -54,7 +55,14 @@ async function getPhotos(albumPath, siteDomain) {
         concurrency: 2,
     });
 
-    return photos.concat(otherPhotos.flat());
+    const allPhotos = photos.concat(otherPhotos.flat());
+
+    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
+    const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
+
+    const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
+
+    return pluckedPhotos;
 }
 
 function scrape(html, site) {
@@ -140,7 +148,7 @@ async function scrapeScene(html, url, site) {
     const poster = videoData.picPreview;
     const trailer = `${videoData.playerOptions.host}${videoData.url}`;
 
-    const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
+    const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
 
     const rawTags = data.keywords.split(', ');
 

src/utils/cookies.js

@@ -0,0 +1,16 @@
+'use strict';
+
+function cookieToData(cookieString) {
+    return cookieString.split('; ').reduce((acc, cookie) => {
+        const [key, value] = cookie.split('=');
+
+        return {
+            ...acc,
+            [key]: value,
+        };
+    }, {});
+}
+
+module.exports = {
+    cookieToData,
+};

src/utils/mofos.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+
+const knex = require('../knex');
+
+async function run() {
+    const network = await knex('networks').where('slug', 'mofos').first();
+    const sites = await knex('sites').where('network_id', network.id);
+
+    await Promise.map(sites, async (site) => {
+        const res = await bhttp.get(site.url);
+
+        console.log(site.url, res.statusCode);
+    }, {
+        concurrency: 5,
+    });
+}
+
+run();

src/utils/upsert.js

@@ -35,7 +35,7 @@ async function upsert(table, items, duplicatesById, identifier = 'id', knex) {
         return Promise.all([
             knex(table).insert(insert),
             knex.transaction(async trx => Promise.all(update.map(item => trx
-                .where({ id: item.id })
+                .where({ [identifier]: item[identifier] })
                 .update(item)
                 .into(table)))),
         ]);