Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

assets/components/actor/actor.vue

@@ -6,20 +6,16 @@
         <FilterBar :fetch-releases="fetchReleases" />
 
         <div class="header">
-            <h2 class="title">{{ actor.name }}</h2>
-        </div>
-
-        <div class="content-inner">
-            <div class="avatars">
             <img
-                    v-for="avatar in actor.avatars"
-                    :key="`avatar-${avatar.id}`"
-                    :src="`/media/${avatar.path}`"
+                v-if="actor.avatars && actor.avatars.length > 0"
+                :src="`/media/${actor.avatars[0].path}`"
                 class="avatar"
             >
-            </div>
 
-            <ul class="bio">
+            <ul class="bio sidebar nolist">
+                <h2 class="title">{{ actor.name }}</h2>
+
+
                 <li v-if="actor.aliases.length">
                     <dfn class="bio-heading">Also known as</dfn>
                     <span>{{ actor.aliases.join(', ') }}</span>
@@ -78,12 +74,31 @@
 
                 <li v-if="actor.social && actor.social.length > 0">
                     <dfn class="bio-heading">Social</dfn>
-                    {{ actor.social.join(',') }}
+                    <a
+                        v-for="social in actor.social"
+                        :key="`social-${social.id}`"
+                        :href="social.url"
+                        target="_blank"
+                        rel="noopener noreferrer"
+                        class="social"
+                    >{{ social.platform || social.url }}</a>
                 </li>
+
+                <li class="description">{{ actor.description }}</li>
+
+                <div class="photos">
+                    <img
+                        v-for="avatar in actor.avatars.slice(1)"
+                        :key="`avatar-${avatar.id}`"
+                        :src="`/media/${avatar.path}`"
+                        class="photo"
+                    >
+                </div>
             </ul>
 
-            <span class="description">{{ actor.description }}</span>
+        </div>
 
+        <div class="content-inner">
             <Releases
                 :releases="releases"
                 :context="actor.name"
@@ -170,13 +185,18 @@ export default {
     padding: 1rem;
 }
 
-.avatars {
-    padding: 1rem;
-}
-
 .avatar {
     height: 20rem;
     margin: 0 1rem 0 0;
+    display: block;
+}
+
+.photo {
+    height: 10rem;
+}
+
+.social {
+    display: block;
 }
 
 .flag {
@@ -184,4 +204,8 @@ export default {
     border: solid 1px $shadow-weak;
     margin: 0 .25rem 0 0;
 }
+
+.releases {
+    flex-grow: 1;
+}
 </style>

migrations/20190325001339_releases.js

@@ -226,6 +226,20 @@ exports.up = knex => Promise.resolve()
         table.datetime('created_at')
             .defaultTo(knex.fn.now());
     }))
+    .then(() => knex.schema.createTable('social', (table) => {
+        table.increments('id', 16);
+
+        table.string('url');
+        table.string('platform');
+
+        table.string('domain');
+        table.integer('target_id', 16);
+
+        table.unique(['url', 'domain', 'target_id']);
+
+        table.datetime('created_at')
+            .defaultTo(knex.fn.now());
+    }))
     .then(() => knex.schema.createTable('actors_associated', (table) => {
         table.increments('id', 16);
 
@@ -281,6 +295,7 @@ exports.down = knex => Promise.resolve()
     .then(() => knex.schema.dropTable('tags'))
     .then(() => knex.schema.dropTable('tags_groups'))
     .then(() => knex.schema.dropTable('media'))
+    .then(() => knex.schema.dropTable('social'))
     .then(() => knex.schema.dropTable('actors'))
     .then(() => knex.schema.dropTable('releases'))
     .then(() => knex.schema.dropTable('sites'))

public/css/style.css

@@ -599,18 +599,25 @@
 .description[data-v-677a8360] {
   padding: 1rem;
 }
-.avatars[data-v-677a8360] {
-  padding: 1rem;
-}
 .avatar[data-v-677a8360] {
   height: 20rem;
   margin: 0 1rem 0 0;
+  display: block;
+}
+.photo[data-v-677a8360] {
+  height: 10rem;
+}
+.social[data-v-677a8360] {
+  display: block;
 }
 .flag[data-v-677a8360] {
   height: 1rem;
   border: solid 1px rgba(0, 0, 0, 0.2);
   margin: 0 .25rem 0 0;
 }
+.releases[data-v-677a8360] {
+  flex-grow: 1;
+}
 
 /* $primary: #ff886c; */
 .header[data-v-80991bcc] {

seeds/02_tags.js

@@ -1,5 +1,3 @@
-'use strict';
-
 const upsert = require('../src/utils/upsert');
 
 const groups = [
@@ -61,7 +59,7 @@ function getTags(groupsMap) {
             name: '69',
             slug: '69',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'airtight',
@@ -69,7 +67,7 @@ function getTags(groupsMap) {
             alias_for: null,
             description: 'A cock in every penetrable hole (of a woman); one in the mouth, one in the vagina, and one in the asshole.',
             priority: 9,
-            group_id: groupsMap['penetration'],
+            group_id: groupsMap.penetration,
         },
         {
             name: 'amateur',
@@ -80,13 +78,13 @@ function getTags(groupsMap) {
             name: 'american',
             slug: 'american',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'anal creampie',
             slug: 'anal-creampie',
             alias_for: null,
-            description: 'Ejaculating into the asshole.'
+            description: 'Ejaculating into the asshole.',
         },
         {
             name: 'anal',
@@ -120,13 +118,13 @@ function getTags(groupsMap) {
             name: 'asian',
             slug: 'asian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'athletic',
             slug: 'athletic',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'ass to mouth',
@@ -148,13 +146,13 @@ function getTags(groupsMap) {
             name: 'ballerina',
             slug: 'ballerina',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'bathroom',
             slug: 'bathroom',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
         {
             name: 'BDSM',
@@ -165,25 +163,25 @@ function getTags(groupsMap) {
             name: 'BBC',
             slug: 'bbc',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big cock',
             slug: 'big-cock',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big butt',
             slug: 'big-butt',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'big boobs',
             slug: 'big-boobs',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'bisexual',
@@ -194,13 +192,13 @@ function getTags(groupsMap) {
             name: 'black hair',
             slug: 'black-hair',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'blonde',
             slug: 'blonde',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'blowjob',
@@ -211,7 +209,7 @@ function getTags(groupsMap) {
             name: 'blowbang',
             slug: 'blowbang',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'bondage',
@@ -222,7 +220,7 @@ function getTags(groupsMap) {
             name: 'brunette',
             slug: 'brunette',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'bukkake',
@@ -233,7 +231,7 @@ function getTags(groupsMap) {
             name: 'cheerleader',
             slug: 'cheerleader',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'choking',
@@ -325,13 +323,13 @@ function getTags(groupsMap) {
             name: 'dress',
             slug: 'dress',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'ebony',
             slug: 'ebony',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'electric shock',
@@ -347,19 +345,19 @@ function getTags(groupsMap) {
             name: 'European',
             slug: 'european',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'facefuck',
             slug: 'facefuck',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'facesitting',
             slug: 'facesitting',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'facial',
@@ -390,7 +388,7 @@ function getTags(groupsMap) {
             name: 'FMF threesome',
             slug: 'fmf',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'gag',
@@ -402,7 +400,7 @@ function getTags(groupsMap) {
             slug: 'gangbang',
             alias_for: null,
             priority: 9,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'gapes',
@@ -424,7 +422,7 @@ function getTags(groupsMap) {
             name: 'hairy',
             slug: 'hairy',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'hardcore',
@@ -435,13 +433,13 @@ function getTags(groupsMap) {
             name: 'high heels',
             slug: 'high-heels',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'hungarian',
             slug: 'hungarian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'humiliation',
@@ -493,13 +491,13 @@ function getTags(groupsMap) {
             name: 'lingerie',
             slug: 'lingerie',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'maid',
             slug: 'maid',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'masturbation',
@@ -510,31 +508,31 @@ function getTags(groupsMap) {
             name: 'MILF',
             slug: 'milf',
             alias_for: null,
-            group_id: groupsMap['age'],
+            group_id: groupsMap.age,
         },
         {
             name: 'MFM threesome',
             slug: 'mfm',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'miniskirt',
             slug: 'miniskirt',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'missionary',
             slug: 'missionary',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'natural boobs',
             slug: 'natural-boobs',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'nipple clamps',
@@ -550,13 +548,13 @@ function getTags(groupsMap) {
             name: 'orgy',
             slug: 'orgy',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'outdoors',
             slug: 'outdoors',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
         {
             name: 'outie pussy',
@@ -597,7 +595,7 @@ function getTags(groupsMap) {
             name: 'redhead',
             slug: 'redhead',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'reverse cowgirl',
@@ -618,7 +616,7 @@ function getTags(groupsMap) {
             name: 'russian',
             slug: 'russian',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'saliva',
@@ -629,7 +627,7 @@ function getTags(groupsMap) {
             name: 'schoolgirl',
             slug: 'schoolgirl',
             alias_for: null,
-            group_id: groupsMap['roleplay'],
+            group_id: groupsMap.roleplay,
         },
         {
             name: 'shaved',
@@ -640,19 +638,19 @@ function getTags(groupsMap) {
             name: 'shoes on',
             slug: 'shoes-on',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'short hair',
             slug: 'short-hair',
             alias_for: null,
-            group_id: groupsMap['hair'],
+            group_id: groupsMap.hair,
         },
         {
             name: 'skirt',
             slug: 'skirt',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'slapping',
@@ -668,7 +666,7 @@ function getTags(groupsMap) {
             name: 'socks',
             slug: 'socks',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'spanking',
@@ -679,7 +677,7 @@ function getTags(groupsMap) {
             name: 'spooning',
             slug: 'spooning',
             alias_for: null,
-            group_id: groupsMap['position'],
+            group_id: groupsMap.position,
         },
         {
             name: 'strapon',
@@ -715,7 +713,7 @@ function getTags(groupsMap) {
             name: 'stockings',
             slug: 'stockings',
             alias_for: null,
-            group_id: groupsMap['clothing'],
+            group_id: groupsMap.clothing,
         },
         {
             name: 'strap-on dildo',
@@ -736,19 +734,19 @@ function getTags(groupsMap) {
             name: 'tattoo',
             slug: 'tattoo',
             alias_for: null,
-            group_id: groupsMap['body'],
+            group_id: groupsMap.body,
         },
         {
             name: 'threesome',
             slug: 'threesome',
             alias_for: null,
-            group_id: groupsMap['group'],
+            group_id: groupsMap.group,
         },
         {
             name: 'teen',
             slug: 'teen',
             alias_for: null,
-            group_id: groupsMap['age'],
+            group_id: groupsMap.age,
         },
         {
             name: 'titty fuck',
@@ -800,7 +798,7 @@ function getTags(groupsMap) {
             name: 'white',
             slug: 'white',
             alias_for: null,
-            group_id: groupsMap['ethnicity'],
+            group_id: groupsMap.ethnicity,
         },
         {
             name: 'wife',
@@ -811,7 +809,7 @@ function getTags(groupsMap) {
             name: 'office',
             slug: 'office',
             alias_for: null,
-            group_id: groupsMap['location'],
+            group_id: groupsMap.location,
         },
     ];
 }
@@ -820,23 +818,23 @@ function getTagAliases(tagsMap) {
     return [
         {
             name: '2-on-1',
-            alias_for: tagsMap['threesome'],
+            alias_for: tagsMap.threesome,
         },
         {
             name: '2 on 1',
-            alias_for: tagsMap['threesome'],
+            alias_for: tagsMap.threesome,
         },
         {
             name: '3+ on 1',
-            alias_for: tagsMap['gangbang'],
+            alias_for: tagsMap.gangbang,
         },
         {
             name: 'anal sex',
-            alias_for: tagsMap['anal'],
+            alias_for: tagsMap.anal,
         },
         {
             name: 'anal gape',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'anilingus',
@@ -844,7 +842,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'asians',
-            alias_for: tagsMap['asian'],
+            alias_for: tagsMap.asian,
         },
         {
             name: 'anal fingering',
@@ -856,7 +854,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'ass fucking',
-            alias_for: tagsMap['anal'],
+            alias_for: tagsMap.anal,
         },
         {
             name: 'atm',
@@ -864,11 +862,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'bald pussy',
-            alias_for: tagsMap['shaved'],
+            alias_for: tagsMap.shaved,
         },
         {
             name: 'ball gag',
-            alias_for: tagsMap['gag'],
+            alias_for: tagsMap.gag,
         },
         {
             name: 'boob fucking',
@@ -876,19 +874,19 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'mfm',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'fmf',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'ffm',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'bgb',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'big ass',
@@ -896,11 +894,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'big black cock',
-            alias_for: tagsMap['bbc'],
+            alias_for: tagsMap.bbc,
         },
         {
             name: 'big black cocks',
-            alias_for: tagsMap['bbc'],
+            alias_for: tagsMap.bbc,
         },
         {
             name: 'big cocks',
@@ -924,39 +922,47 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'bi',
-            alias_for: tagsMap['bisexual'],
+            alias_for: tagsMap.bisexual,
         },
         {
             name: 'black',
-            alias_for: tagsMap['ebony'],
+            alias_for: tagsMap.ebony,
         },
         {
             name: 'blonde hair',
-            alias_for: tagsMap['blonde'],
+            alias_for: tagsMap.blonde,
         },
         {
             name: 'blondes',
-            alias_for: tagsMap['blonde'],
+            alias_for: tagsMap.blonde,
         },
         {
             name: 'blow job',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjobs',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjob pov',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'blowjob (double)',
             alias_for: tagsMap['double-blowjob'],
         },
+        {
+            name: 'blowjob - double',
+            alias_for: tagsMap['double-blowjob'],
+        },
         {
             name: 'blowjob (pov)',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
+        },
+        {
+            name: 'blowjob - pov',
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'boob job',
@@ -968,11 +974,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'brown hair',
-            alias_for: tagsMap['brunette'],
+            alias_for: tagsMap.brunette,
         },
         {
             name: 'brunettes',
-            alias_for: tagsMap['brunette'],
+            alias_for: tagsMap.brunette,
         },
         {
             name: 'buttplug',
@@ -992,7 +998,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'cheer leader',
-            alias_for: tagsMap['cheerleader'],
+            alias_for: tagsMap.cheerleader,
         },
         {
             name: 'clover clamps',
@@ -1000,11 +1006,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'couples fantasies',
-            alias_for: tagsMap['couples'],
+            alias_for: tagsMap.couples,
         },
         {
             name: 'creampies',
-            alias_for: tagsMap['creampie'],
+            alias_for: tagsMap.creampie,
         },
         {
             name: 'crop', // a type of whip, not [sic] short for corporal
@@ -1028,11 +1034,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'cum swallowing',
-            alias_for: tagsMap['swallowing'],
+            alias_for: tagsMap.swallowing,
         },
         {
             name: 'cum shot',
-            alias_for: tagsMap['cumshot'],
+            alias_for: tagsMap.cumshot,
         },
         {
             name: 'cunnilingus',
@@ -1044,15 +1050,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'deep throat',
-            alias_for: tagsMap['deepthroat'],
+            alias_for: tagsMap.deepthroat,
         },
         {
             name: 'deepthroating',
-            alias_for: tagsMap['deepthroat'],
+            alias_for: tagsMap.deepthroat,
         },
         {
             name: 'dildo',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'doggystyle',
@@ -1072,15 +1078,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'dom',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'domination',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'dominatrix',
-            alias_for: tagsMap['femdom'],
+            alias_for: tagsMap.femdom,
         },
         {
             name: 'dp',
@@ -1090,6 +1096,10 @@ function getTagAliases(tagsMap) {
             name: 'double penetration (dp)',
             alias_for: tagsMap['double-penetration'],
         },
+        {
+            name: 'double penetration - dp',
+            alias_for: tagsMap['double-penetration'],
+        },
         {
             name: 'dap',
             alias_for: tagsMap['double-anal'],
@@ -1132,7 +1142,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'drool',
-            alias_for: tagsMap['saliva'],
+            alias_for: tagsMap.saliva,
         },
         {
             name: 'enhanced',
@@ -1144,27 +1154,27 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'facefucking',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face fuck',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face fucking',
-            alias_for: tagsMap['facefuck'],
+            alias_for: tagsMap.facefuck,
         },
         {
             name: 'face sitting',
-            alias_for: tagsMap['facesitting'],
+            alias_for: tagsMap.facesitting,
         },
         {
             name: 'facial cumshot',
-            alias_for: tagsMap['facial'],
+            alias_for: tagsMap.facial,
         },
         {
             name: 'facials',
-            alias_for: tagsMap['facial'],
+            alias_for: tagsMap.facial,
         },
         {
             name: 'fake boobs',
@@ -1180,27 +1190,27 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'foot fetish',
-            alias_for: tagsMap['feet'],
+            alias_for: tagsMap.feet,
         },
         {
             name: 'french kissing',
-            alias_for: tagsMap['kissing'],
+            alias_for: tagsMap.kissing,
         },
         {
             name: 'gape',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'gaping',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'gapes (gaping asshole)',
-            alias_for: tagsMap['gapes'],
+            alias_for: tagsMap.gapes,
         },
         {
             name: 'group sex',
-            alias_for: tagsMap['orgy'],
+            alias_for: tagsMap.orgy,
         },
         {
             name: 'flagellation',
@@ -1212,7 +1222,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'huge toys',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'innie',
@@ -1224,23 +1234,23 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'lezdom',
-            alias_for: tagsMap['lesbian'],
+            alias_for: tagsMap.lesbian,
         },
         {
             name: 'mini-skirt',
-            alias_for: tagsMap['miniskirt'],
+            alias_for: tagsMap.miniskirt,
         },
         {
             name: 'mmf',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'mff',
-            alias_for: tagsMap['fmf'],
+            alias_for: tagsMap.fmf,
         },
         {
             name: 'mature & milf',
-            alias_for: tagsMap['milf'],
+            alias_for: tagsMap.milf,
         },
         {
             name: 'natural',
@@ -1252,7 +1262,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'oral',
-            alias_for: tagsMap['blowjob'],
+            alias_for: tagsMap.blowjob,
         },
         {
             name: 'outie',
@@ -1260,11 +1270,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'piercing',
-            alias_for: tagsMap['piercings'],
+            alias_for: tagsMap.piercings,
         },
         {
             name: 'pierced',
-            alias_for: tagsMap['piercings'],
+            alias_for: tagsMap.piercings,
         },
         {
             name: 'prolapse',
@@ -1284,11 +1294,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'red hair',
-            alias_for: tagsMap['redhead'],
+            alias_for: tagsMap.redhead,
         },
         {
             name: 'red head',
-            alias_for: tagsMap['redhead'],
+            alias_for: tagsMap.redhead,
         },
         {
             name: 'rimming',
@@ -1300,39 +1310,39 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'role play',
-            alias_for: tagsMap['roleplay'],
+            alias_for: tagsMap.roleplay,
         },
         {
             name: 'rope bondage',
-            alias_for: tagsMap['bondage'],
+            alias_for: tagsMap.bondage,
         },
         {
             name: 'rough sex',
-            alias_for: tagsMap['rough'],
+            alias_for: tagsMap.rough,
         },
         {
             name: 'school girl',
-            alias_for: tagsMap['schoolgirl'],
+            alias_for: tagsMap.schoolgirl,
         },
         {
             name: 'sadomasochism',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'sadism',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'scissoring',
-            alias_for: tagsMap['lesbian'],
+            alias_for: tagsMap.lesbian,
         },
         {
             name: 'sex toys',
-            alias_for: tagsMap['toys'],
+            alias_for: tagsMap.toys,
         },
         {
             name: 'shaved pussy',
-            alias_for: tagsMap['shaved'],
+            alias_for: tagsMap.shaved,
         },
         {
             name: 'shoes',
@@ -1340,7 +1350,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'slave',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'small ass',
@@ -1352,11 +1362,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'spit',
-            alias_for: tagsMap['saliva'],
+            alias_for: tagsMap.saliva,
         },
         {
             name: 'spitroast',
-            alias_for: tagsMap['mfm'],
+            alias_for: tagsMap.mfm,
         },
         {
             name: 'standing doggystyle',
@@ -1364,7 +1374,7 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'swallow',
-            alias_for: tagsMap['swallowing'],
+            alias_for: tagsMap.swallowing,
         },
         {
             name: 'strap-on',
@@ -1380,19 +1390,19 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'sub',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'submission',
-            alias_for: tagsMap['bdsm'],
+            alias_for: tagsMap.bdsm,
         },
         {
             name: 'tattoos',
-            alias_for: tagsMap['tattoo'],
+            alias_for: tagsMap.tattoo,
         },
         {
             name: 'teens',
-            alias_for: tagsMap['teen'],
+            alias_for: tagsMap.teen,
         },
         {
             name: 'tiny boobs',
@@ -1412,15 +1422,15 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'trans',
-            alias_for: tagsMap['transsexual'],
+            alias_for: tagsMap.transsexual,
         },
         {
             name: 'trimmed pussy',
-            alias_for: tagsMap['trimmed'],
+            alias_for: tagsMap.trimmed,
         },
         {
             name: 'ts',
-            alias_for: tagsMap['transsexual'],
+            alias_for: tagsMap.transsexual,
         },
         {
             name: 'whipping',
@@ -1428,11 +1438,11 @@ function getTagAliases(tagsMap) {
         },
         {
             name: 'work',
-            alias_for: tagsMap['office'],
+            alias_for: tagsMap.office,
         },
         {
             name: 'workplace',
-            alias_for: tagsMap['office'],
+            alias_for: tagsMap.office,
         },
         {
             name: 'zapper',

seeds/04_countries.js

@@ -1,8 +1,6 @@
-'use strict';
+const upsert = require('../src/utils/upsert');
 
-exports.seed = knex => knex('countries')
-    .delete()
-    .then(() => knex('countries').insert([
+const countries = [
     {
         name: 'Afghanistan',
         code: '004',
@@ -1248,4 +1246,12 @@ exports.seed = knex => knex('countries')
         code: '716',
         alpha2: 'ZW',
     },
-    ]));
+];
+
+exports.seed = knex => knex('countries')
+    .then(async () => {
+        const duplicates = await knex('countries').select('*');
+        const duplicatesByAlpha2 = duplicates.reduce((acc, country) => ({ ...acc, [country.alpha2]: country }), {});
+
+        return upsert('countries', countries, duplicatesByAlpha2, 'alpha2', knex);
+    });

src/actors.js

@@ -9,11 +9,13 @@ const whereOr = require('./utils/where-or');
 const { createActorMediaDirectory, storeAvatars } = require('./media');
 
 async function curateActor(actor) {
-    const [aliases, avatars] = await Promise.all([
+    const [aliases, avatars, social] = await Promise.all([
         knex('actors').where({ alias_for: actor.id }),
         knex('media')
             .where({ domain: 'actors', target_id: actor.id })
             .orderBy('index'),
+        knex('social')
+            .where({ domain: 'actors', target_id: actor.id }),
     ]);
 
     return {
@@ -46,6 +48,7 @@ async function curateActor(actor) {
         aliases: aliases.map(({ name }) => name),
         slug: actor.slug,
         avatars,
+        social,
     };
 }
 
@@ -94,6 +97,35 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
     return curatedActor;
 }
 
+function curateSocialEntry(url, actor) {
+    const { hostname, origin, pathname } = new URL(url);
+    const platform = ['twitter', 'instagram', 'snapchat', 'modelhub', 'youtube'].find(platformName => hostname.match(platformName));
+
+    return {
+        url: `${origin}${pathname}`,
+        platform,
+        domain: 'actors',
+        target_id: actor.id,
+    };
+}
+
+function curateSocialEntries(urls, actor) {
+    if (!urls) {
+        return [];
+    }
+
+    return urls.reduce((acc, url) => {
+        const socialEntry = curateSocialEntry(url, actor);
+
+        if (acc.some(entry => socialEntry.url === entry.url)) {
+            // prevent duplicates
+            return acc;
+        }
+
+        return [...acc, socialEntry];
+    }, []);
+}
+
 async function fetchActors(queryObject) {
     const releases = await knex('actors')
         .select(
@@ -112,34 +144,30 @@ async function fetchActors(queryObject) {
 async function storeActor(actor, scraped = false, scrapeSuccess = false) {
     const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
 
-    const actorEntries = await knex('actors')
+    const [actorEntry] = await knex('actors')
         .insert(curatedActor)
         .returning('*');
 
-    if (actorEntries.length) {
-        const actorEntry = actorEntries[0];
+    await knex('social').insert(curateSocialEntries(actor.social, actor));
 
     console.log(`Added new entry for actor '${actor.name}'`);
 
     return actorEntry;
-    }
-
-    console.error(`Unable to save profile for '${actor.name}'`);
-
-    return null;
 }
 
 async function updateActor(actor, scraped = false, scrapeSuccess = false) {
     const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
 
-    const actorEntries = await knex('actors')
+    const [actorEntry] = await knex('actors')
         .where({ id: actor.id })
         .update(curatedActor)
         .returning('*');
 
+    await knex('social').insert(curateSocialEntries(actor.social, actor));
+
     console.log(`Updated entry for actor '${actor.name}'`);
 
-    return actorEntries[0];
+    return actorEntry;
 }
 
 function mergeProfiles(profiles, actor) {
@@ -177,7 +205,6 @@ function mergeProfiles(profiles, actor) {
     }, {
         social: [],
         avatars: [],
-        ...actor,
     });
 }
 

src/scrapers/brazzers.js

@@ -148,7 +148,7 @@ function scrapeProfile(html, url, actorName) {
     };
 
     if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity;
-    if (bio.Measurements && bio.Measurements.match(/\w+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
+    if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
     if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate();
     if (bio['Birth Location']) profile.birthPlace = bio['Birth Location'];
     if (bio['Pussy Type']) profile.pussy = bio['Pussy Type'].split(',').slice(-1)[0].toLowerCase();

src/scrapers/mofos.js

@@ -1,67 +1,84 @@
 'use strict';
 
+const Promise = require('bluebird');
 const bhttp = require('bhttp');
-const cheerio = require('cheerio');
+const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
 const moment = require('moment');
 
-const knex = require('../knex');
+const { fetchSites } = require('../sites');
+const { cookieToData } = require('../utils/cookies');
 const { matchTags } = require('../tags');
 
+function getThumbs(scene) {
+    if (scene.images.poster) {
+        return scene.images.poster.map(image => image.xl.url);
+    }
+
+    if (scene.images.card_main_rect) {
+        return scene.images.card_main_rect
+            .concat(scene.images.card_secondary_rect || [])
+            .map(image => image.xl.url.replace('.thumb', ''));
+    }
+
+    return [];
+}
+
 /* eslint-disable newline-per-chained-call */
-function scrape(html, site) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElements = $('.widget-release-card').toArray();
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
+        const url = `https://www.mofos.com/scene/${entryId}/`;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
 
-    return sceneElements.map((element) => {
-        const sceneLinkElement = $(element).find('.title a');
+        const rawTags = data.tags.map(tag => tag.name);
+        const tags = await matchTags(rawTags);
 
-        const title = sceneLinkElement.text().trim();
-        const url = `https://www.mofos.com${sceneLinkElement.attr('href')}`;
-        const entryId = url.split('/').slice(-2, -1)[0];
-
-        const date = moment.utc($(element).find('.date-added').text(), 'MMM DD, YYYY').toDate();
-        const actors = $(element).find('.girls-name a').map((actorIndex, actorElement) => $(actorElement).attr('title').replace(/\s+/g, ' ')).toArray();
-
-        const stars = Number($(element).find('.rating').text().slice(0, -1).trim()) / 20;
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
         return {
             url,
             entryId,
             title,
+            description,
             actors,
-            date,
-            rating: {
-                stars,
+            tags,
+            poster,
+            photos,
+            trailer: {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
             },
+            date,
             site,
         };
-    });
+    }));
 }
 
-async function scrapeScene(html, url, site) {
-    const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElement = $('.video-info');
+async function scrapeScene(data, url, site) {
+    const { id: entryId, title, description } = data;
+    const date = new Date(data.dateReleased);
+    const actors = data.actors.map(actor => actor.name);
 
-    const entryId = url.split('/').slice(-2, -1)[0];
-    const title = sceneElement.find('.title').text();
-    const description = sceneElement.find('.desc').text();
-    const actors = sceneElement.find('.girls-site-box a.model-name').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
+    const rawTags = data.tags.map(tag => tag.name);
 
-    const siteElement = sceneElement.find('.site-name');
-    const sitename = siteElement.text().trim();
-    const siteId = sitename.replace(/\s+/g, '').toLowerCase();
-    const siteUrl = siteElement.attr('href').split('/').slice(0, 4).join('/');
+    const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
-    const stars = Number(sceneElement.find('.rating-box .rating').text().slice(0, -1).trim()) / 20;
+    const siteName = data.collections[0].name;
+    const siteId = data.collections[0].id;
+    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
+    const siteUrl = `https://www.mofos.com/scenes?site=${siteId}`;
 
-    const rawTags = sceneElement.find('.categories a').map((tagIndex, tagElement) => $(tagElement).text().trim()).toArray();
-
-    const [channelSite, tags] = await Promise.all([
-        knex('sites')
-            .where({ slug: siteId })
-            .orWhere({ url: `https://www.mofos.com${siteUrl}` })
-            .orWhere({ name: sitename })
-            .first(),
+    const [[channelSite], tags] = await Promise.all([
+        site.isFallback
+            ? fetchSites({
+                slug: siteSlug,
+                name: siteName,
+                url: siteUrl,
+            })
+            : [site],
         matchTags(rawTags),
     ]);
 
@@ -72,35 +89,61 @@ async function scrapeScene(html, url, site) {
         description,
         actors,
         tags,
-        rating: {
-            stars,
+        poster,
+        photos,
+        trailer: {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
         },
-        site: channelSite || site,
+        date,
+        site: channelSite,
     };
 }
 
 async function fetchLatest(site, page = 1) {
-    const res = page > 1
-        ? await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/${page}/`)
-        : await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/`); // explicit page 1 redirects to homepage
+    const { search } = new URL(site.url);
+    const siteId = new URLSearchParams(search).get('site');
 
-    return scrape(res.body.toString(), site);
-}
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
 
-async function fetchUpcoming(site) {
-    const res = await bhttp.get(`${site.url}/all-models/all-categories/upcoming/bydate/`);
+    await session.get(site.url);
 
-    return scrape(res.body.toString(), site);
+    const cookieString = await cookieJar.getCookieStringAsync(site.url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
 }
 
 async function fetchScene(url, site) {
-    const res = await bhttp.get(url);
+    const entryId = url.match(/\d+/)[0];
 
-    return scrapeScene(res.body.toString(), url, site);
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeScene(res.body.result, url, site);
 }
 
 module.exports = {
     fetchLatest,
-    fetchUpcoming,
     fetchScene,
 };

src/scrapers/pornhub.js

@@ -59,7 +59,7 @@ async function scrapeProfile(html, _url, actorName) {
         profile.residenceCountry = residenceCountryEntry ? residenceCountryEntry.alpha2 : null;
     }
 
-    if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-');
+    if (bio.Measurements && bio.Measurements !== '--') [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-').map(measurement => parseInt(measurement, 10) || null);
     if (bio['Fake Boobs']) profile.naturalBoobs = bio['Fake Boobs'] === 'No';
 
     if (bio.Height) profile.height = Number(bio.Height.match(/\(\d+/)[0].slice(1));

src/scrapers/realitykings.js

@@ -4,8 +4,10 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
-const { JSDOM } = require('jsdom');
+const moment = require('moment');
 
+const { fetchSites } = require('../sites');
+const { cookieToData } = require('../utils/cookies');
 const { matchTags } = require('../tags');
 
 function getThumbs(scene) {
@@ -22,40 +24,18 @@ function getThumbs(scene) {
     return [];
 }
 
-async function scrapeLatest(html, site) {
-    const { document } = new JSDOM(html).window;
-
-    const scriptString = document.querySelector('script').textContent;
-    const prefix = 'window.__JUAN.initialState = {';
-
-    const dataStart = scriptString.slice(scriptString.indexOf(prefix) + prefix.length - 1);
-    const dataString = dataStart.slice(0, dataStart.indexOf('};') + 1);
-    const data = JSON.parse(dataString);
-
-    const actorsMap = data.entities.actors;
-    const tagsMap = data.entities.tags;
-    const scenes = Object.values(data.entities.releases);
-
-    return Promise.all(scenes.map(async (scene) => {
-        const {
-            id: entryId,
-            title,
-            description,
-        } = scene;
-
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
         const url = `https://www.realitykings.com/scene/${entryId}/`;
-        const date = new Date(scene.dateReleased);
-        const actors = scene.actors.map(actorId => actorsMap[actorId].name);
-        const duration = scene.videos.mediabook && scene.videos.mediabook.length;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
 
-        const rawTags = scene.tags.map(tagId => tagsMap[tagId].name);
+        const rawTags = data.tags.map(tag => tag.name);
         const tags = await matchTags(rawTags);
 
-        const [poster, ...photos] = getThumbs(scene);
-        const trailer720p = scene.videos.mediabook && scene.videos.mediabook.files['720p'] && scene.videos.mediabook.files['720p'].urls.view;
-        const trailer360p = scene.videos.mediabook && scene.videos.mediabook.files['360p'] && scene.videos.mediabook.files['360p'].urls.view;
-
-        const { likes, dislikes } = scene.stats;
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
         return {
             url,
@@ -63,41 +43,44 @@ async function scrapeLatest(html, site) {
             title,
             description,
             actors,
-            date,
             tags,
-            duration,
             poster,
             photos,
-            trailer: {
-                src: trailer720p || trailer360p,
-                quality: trailer720p ? 720 : 360,
+            trailer: trailer && {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
             },
-            rating: { likes, dislikes },
+            date,
             site,
         };
     }));
 }
 
 async function scrapeScene(data, url, site) {
-    const {
-        id: entryId,
-        title,
-        description,
-    } = data;
-
+    const { id: entryId, title, description } = data;
     const date = new Date(data.dateReleased);
     const actors = data.actors.map(actor => actor.name);
 
-    const { likes, dislikes } = data.stats;
-
     const rawTags = data.tags.map(tag => tag.name);
-    const tags = await matchTags(rawTags);
 
     const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
-    const duration = data.videos.mediabook && data.videos.mediabook.length;
-    const trailer720p = data.videos.mediabook && data.videos.mediabook.files['720p'] && data.videos.mediabook.files['720p'].urls.view;
-    const trailer360p = data.videos.mediabook && data.videos.mediabook.files['360p'] && data.videos.mediabook.files['360p'].urls.view;
+    const siteName = data.collections[0].name;
+    const siteId = data.collections[0].id;
+    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
+    const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
+
+    const [[channelSite], tags] = await Promise.all([
+        site.isFallback
+            ? fetchSites({
+                slug: siteSlug,
+                name: siteName,
+                url: siteUrl,
+            })
+            : [site],
+        matchTags(rawTags),
+    ]);
 
     return {
         url,
@@ -105,41 +88,56 @@ async function scrapeScene(data, url, site) {
         title,
         description,
         actors,
-        date,
-        duration,
         tags,
         poster,
         photos,
-        trailer: {
-            src: trailer720p || trailer360p,
-            quality: trailer720p ? 720 : 360,
+        trailer: trailer && {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
         },
-        rating: {
-            likes,
-            dislikes,
-        },
-        site,
+        date,
+        site: channelSite,
     };
 }
 
-async function fetchLatest(site, page = 1) {
+function getUrl(site) {
     const { hostname, search } = new URL(site.url);
 
     if (hostname.match(/(www\.)?realitykings\.com/) && search.match(/\?site=\d+/)) {
-        const res = await bhttp.get(`${site.url}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return site.url;
     }
 
     if (site.parameters && site.parameters.siteId) {
-        const res = await bhttp.get(`https://www.realitykings.com/scenes?site=${site.parameters.siteId}&page=${page}`);
-
-        return scrapeLatest(res.body.toString(), site);
+        return `https://www.realitykings.com/scenes?site=${site.parameters.siteId}`;
     }
 
     throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
 }
 
+async function fetchLatest(site, page = 1) {
+    const url = getUrl(site);
+    const { search } = new URL(url);
+    const siteId = new URLSearchParams(search).get('site');
+
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
+}
+
 async function fetchScene(url, site) {
     const entryId = url.match(/\d+/)[0];
 
@@ -148,8 +146,8 @@ async function fetchScene(url, site) {
 
     await session.get(url);
 
-    const cookies = await cookieJar.getCookieStringAsync(url);
-    const instanceToken = cookies.split(';')[0].split('=')[1];
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
 
     const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
         headers: {
@@ -157,7 +155,7 @@ async function fetchScene(url, site) {
         },
     });
 
-    return scrapeScene(res.body.result.parent || res.body.result, url, site);
+    return scrapeScene(res.body.result, url, site);
 }
 
 module.exports = {

src/scrapers/xempire.js

@@ -7,6 +7,7 @@ const moment = require('moment');
 
 const knex = require('../knex');
 const { matchTags } = require('../tags');
+const pluckPhotos = require('../utils/pluck-photos');
 
 const defaultTags = {
     hardx: [],
@@ -36,7 +37,7 @@ function scrapePhotos(html) {
     return unlockedPhotos.concat(lockedThumbnails);
 }
 
-async function getPhotos(albumPath, siteDomain) {
+async function getPhotos(albumPath, siteDomain, site) {
     const albumUrl = `https://${siteDomain}${albumPath}`;
 
     const html = await fetchPhotos(albumUrl);
@@ -54,7 +55,14 @@ async function getPhotos(albumPath, siteDomain) {
         concurrency: 2,
     });
 
-    return photos.concat(otherPhotos.flat());
+    const allPhotos = photos.concat(otherPhotos.flat());
+
+    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
+    const photoIndexes = pluckPhotos(allPhotos.length - 1, photoLimit);
+
+    const pluckedPhotos = photoIndexes.map(photoIndex => allPhotos[photoIndex]);
+
+    return pluckedPhotos;
 }
 
 function scrape(html, site) {
@@ -140,7 +148,7 @@ async function scrapeScene(html, url, site) {
     const poster = videoData.picPreview;
     const trailer = `${videoData.playerOptions.host}${videoData.url}`;
 
-    const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
+    const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain, site);
 
     const rawTags = data.keywords.split(', ');
 

src/utils/cookies.js

@@ -0,0 +1,16 @@
+'use strict';
+
+function cookieToData(cookieString) {
+    return cookieString.split('; ').reduce((acc, cookie) => {
+        const [key, value] = cookie.split('=');
+
+        return {
+            ...acc,
+            [key]: value,
+        };
+    }, {});
+}
+
+module.exports = {
+    cookieToData,
+};

src/utils/mofos.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+
+const knex = require('../knex');
+
+async function run() {
+    const network = await knex('networks').where('slug', 'mofos').first();
+    const sites = await knex('sites').where('network_id', network.id);
+
+    await Promise.map(sites, async (site) => {
+        const res = await bhttp.get(site.url);
+
+        console.log(site.url, res.statusCode);
+    }, {
+        concurrency: 5,
+    });
+}
+
+run();

src/utils/upsert.js

@@ -35,7 +35,7 @@ async function upsert(table, items, duplicatesById, identifier = 'id', knex) {
         return Promise.all([
             knex(table).insert(insert),
             knex.transaction(async trx => Promise.all(update.map(item => trx
-                .where({ id: item.id })
+                .where({ [identifier]: item[identifier] })
                 .update(item)
                 .into(table)))),
         ]);