Replaced network and tag files with SQLite database.

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 dist/
 config/*
 !config/default.js
+db.sqlite

config/default.js

@@ -2,9 +2,23 @@
 
 module.exports = {
     include: [
-        'xempire',
         'julesjordan',
+        ['kink', [
+            'boundgangbangs',
+            'brutalsessions',
+            'devicebondage',
+            'everythingbutt',
+            'familiestied',
+            'fuckingmachines',
+            'hogtied',
+            'publicdisgrace',
+            'sexandsubmission',
+            'theupperfloor',
+            'waterbondage',
+        ]],
+        'legalporno',
         'pervcity',
+        'xempire',
     ],
     columns: [
         {

knexfile.js

@@ -0,0 +1,9 @@
+// Update with your config settings.
+
+module.exports = {
+    client: 'sqlite3',
+    connection: {
+        filename: './db.sqlite',
+    },
+    useNullAsDefault: true,
+};

migrations/20190325001339_releases.js

@@ -0,0 +1,103 @@
+'use strict';
+
+exports.up = knex => Promise.resolve()
+    .then(() => knex.schema.createTable('actors', (table) => {
+        table.increments('id', 8);
+
+        table.string('name');
+        table.integer('alias_for', 8)
+            .references('id')
+            .inTable('actors');
+    }))
+    .then(() => knex.schema.createTable('tags', (table) => {
+        table.string('tag', 20)
+            .primary();
+
+        table.string('alias_for', 20)
+            .references('tag')
+            .inTable('tags');
+    }))
+    .then(() => knex.schema.createTable('networks', (table) => {
+        table.string('id', 32)
+            .primary();
+
+        table.string('name');
+        table.string('url');
+        table.string('description');
+    }))
+    .then(() => knex.schema.createTable('sites', (table) => {
+        table.string('id', 32)
+            .primary();
+
+        table.string('label', 6);
+
+        table.string('network_id', 32)
+            .notNullable()
+            .references('id')
+            .inTable('networks');
+
+        table.string('name');
+        table.string('url');
+        table.string('description');
+    }))
+    .then(() => knex.schema.createTable('releases', (table) => {
+        table.increments('id', 12);
+
+        table.string('site_id', 32)
+            .notNullable()
+            .references('id')
+            .inTable('sites');
+
+        table.string('shoot_id');
+        table.string('title');
+        table.date('date');
+        table.text('description');
+
+        table.integer('duration')
+            .unsigned();
+
+        table.integer('likes')
+            .unsigned();
+
+        table.integer('dislikes')
+            .unsigned();
+
+        table.integer('rating')
+            .unsigned();
+    }))
+    .then(() => knex.schema.createTable('actors_associated', (table) => {
+        table.increments('id', 16);
+
+        table.integer('release_id', 12)
+            .notNullable()
+            .references('id')
+            .inTable('releases');
+
+        table.integer('actor_id', 8)
+            .notNullable()
+            .references('id')
+            .inTable('actors');
+    }))
+    .then(() => knex.schema.createTable('tags_associated', (table) => {
+        table.string('tag_id', 20)
+            .notNullable()
+            .references('tag')
+            .inTable('tags');
+
+        table.string('site_id')
+            .references('id')
+            .inTable('sites');
+
+        table.string('release_id')
+            .references('id')
+            .inTable('releases');
+    }));
+
+exports.down = knex => Promise.resolve()
+    .then(() => knex.schema.dropTable('tags_associated'))
+    .then(() => knex.schema.dropTable('actors_associated'))
+    .then(() => knex.schema.dropTable('releases'))
+    .then(() => knex.schema.dropTable('sites'))
+    .then(() => knex.schema.dropTable('networks'))
+    .then(() => knex.schema.dropTable('actors'))
+    .then(() => knex.schema.dropTable('tags'));

package.json

@@ -6,7 +6,13 @@
     "scripts": {
         "start": "node src/app.js",
         "eslint": "eslint src/",
-        "eslint-watch": "esw --watch src/"
+        "eslint-watch": "esw --watch src/",
+        "knex": "knex",
+        "migrate-make": "knex-migrate generate",
+        "migrate": "knex-migrate up",
+        "rollback": "knex-migrate down",
+        "seed-make": "knex seed:make",
+        "seed": "knex seed:run"
     },
     "repository": {
         "type": "git",
@@ -36,10 +42,13 @@
         "cheerio": "^1.0.0-rc.2",
         "clipboardy": "^1.2.3",
         "config": "^3.0.1",
+        "knex": "^0.16.3",
+        "knex-migrate": "^1.7.1",
         "moment": "^2.24.0",
         "neo-blessed": "^0.2.0",
         "node-fetch": "^2.3.0",
         "opn": "^5.4.0",
+        "sqlite3": "^4.0.6",
         "tty-table": "^2.7.0",
         "yargs": "^13.2.2"
     }

seeds/networks.js

@@ -0,0 +1,36 @@
+'use strict';
+
+/* eslint-disable max-len */
+exports.seed = knex => Promise.resolve()
+    .then(() => knex('networks').del())
+    .then(() => knex('networks').insert([
+        {
+            id: 'julesjordan',
+            name: 'Jules Jordan',
+            url: 'https://www.julesjordan.com',
+        },
+        {
+            id: 'kink',
+            name: 'Kink',
+            url: 'https://www.kink.com',
+            description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
+        },
+        {
+            id: 'legalporno',
+            name: 'LegalPorno',
+            url: 'https://www.legalporno.com',
+            description: 'The Best HD Porn For You!',
+        },
+        {
+            id: 'pervcity',
+            name: 'Perv City',
+            url: 'https://www.pervcity.com',
+            description: '',
+        },
+        {
+            id: 'xempire',
+            name: 'XEmpire',
+            url: 'https://www.xempire.com',
+            description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
+        },
+    ]));

seeds/sites.js

@@ -1,237 +1,355 @@
 'use strict';
 
 /* eslint-disable max-len */
-module.exports = {
+exports.seed = knex => Promise.resolve()
-    name: 'Kink',
+    .then(() => knex('sites').del())
-    url: 'https://www.kink.com',
+    .then(() => knex('sites').insert([
-    description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
+        // JULES JORDAN
-    sites: {
+        {
-        // some sites discontinued and for scene fetching purposes only
+            id: 'julesjordan',
-        kink: {
+            name: 'Jules Jordan',
+            label: 'julesj',
+            url: 'https://www.julesjordan.com',
+            description: 'Jules Jordan\'s Official Membership Site',
+            network_id: 'julesjordan',
+        },
+        // KINK
+        {
+            id: 'kink',
             name: 'Kink',
             label: 'kink',
             url: 'https://www.kink.com',
             description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        thirtyminutesoftorment: {
+        {
+            id: 'thirtyminutesoftorment',
             name: '30 Minutes of Torment',
             label: '30mtor',
             url: 'https://www.kink.com/channel/30minutesoftorment',
             description: 'Thick-Muscled Men Endure 30 Minutes Of BDSM Torment By A Pain-Inducing Dom. Can they take 30 Minutes of Torment? Watch as top gay pornstars take on the challenge of a lifetime. Bondage, BDSM, punishment, huge insertions, & more!',
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        boundgangbangs: {
+        {
+            id: 'boundgangbangs',
             name: 'Bound Gangbangs',
             label: 'boundg',
             url: 'https://www.kink.com/channel/boundgangbangs',
             description: 'Poweless whores tied in bondage and stuffed with a cock in every hole. At BoundGangbangs women get surprise extreme gangbangs, blindfolds, deepthroat blowjobs, sex punishment, bondage, double penetration and interracial sex.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        boundgods: {
+        {
+            id: 'boundgods',
             name: 'Bound Gods',
             label: 'bngods',
             url: 'https://www.kink.com/channel/boundgods',
             description: 'Muscle Studs Are Bound, Gagged & Spread For A Deep Cock Pounding. Not even the most rock hard muscled studs can escape punishment & submission on BoundGods.com Watch the hottest studs get tied down, fucked & submitted.',
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        boundinpublic: {
+        {
+            id: 'boundinpublic',
             name: 'Bound in Public',
             label: 'boundp',
             url: 'https://www.kink.com/channel/boundinpublic',
             description: 'Cum Starved Sluts Humiliated And Fucked Hard In Public By Hung Studs.',
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        brutalsessions: {
+        {
+            id: 'brutalsessions',
             name: 'Brutal Sessions',
             label: 'brutal',
             url: 'https://www.kink.com/channel/brutalsessions',
-            description: 'Hardcore BDSM jam packed with XXX fucking in bondage! We\'re taking dungeon sex beyond the castle!',
+            description: "Hardcore BDSM jam packed with XXX fucking in bondage! We're taking dungeon sex beyond the castle!",
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        buttmachineboys: {
+        {
+            id: 'buttmachineboys',
             name: 'Butt Machine Boys',
             label: 'bmboys',
             url: 'https://www.kink.com/channel/buttmachineboys',
             description: 'Powerful Fucking Machines Pound Hot Men Hard & Deep.',
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        devicebondage: {
+        {
+            id: 'devicebondage',
             name: 'Device Bondage',
             label: 'dvbond',
             url: 'https://www.kink.com/channel/devicebondage',
             description: 'The Domination Of Sluts In Barbaric Metal Devices. Device Bondage takes BDSM porn to new levels with extreme restraints & unique devices with beautiful pornstars to huge, forced squirting orgasms.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        devinebitches: {
+        {
+            id: 'devinebitches',
             name: 'Devine Bitches',
             label: 'dbitch',
             url: 'https://www.kink.com/channel/divinebitches',
             description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with  obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.',
-            tags: ['bdsm', 'femdom'],
+            network_id: 'kink',
         },
-        electrosluts: {
+        {
+            id: 'electrosluts',
             name: 'Electrosluts',
             label: 'esluts',
             url: 'https://www.kink.com/channel/electrosluts',
             description: 'Lezdoms Take Submissive Sluts To Their Limits, Shocking & Tormenting Their Wet Hot Pussies. Pornstars live out their electric bondage fantasies while dominatrixes use electrodes, paddles, caddle prods, & more to bring them to intense orgasms!',
-            tags: ['bdsm', 'lesbian'],
+            network_id: 'kink',
         },
-        everythingbutt: {
+        {
+            id: 'everythingbutt',
             name: 'Everything Butt',
             label: 'evbutt',
             url: 'https://www.kink.com/channel/everythingbutt',
             description: 'Gaping Anal Holes Are Stuffed & Stretched To The Max. Anal Fisting, Enemas & Rimming Has Never Tasted So Good. EverythingButt.com explores the extreme limits of FemDom lesbian anal. Watch asses get destroyed by brutal fistings, huge insertions, double anal & more!',
-            tags: ['bdsm', 'lesbian'],
+            network_id: 'kink',
         },
-        familiestied: {
+        {
+            id: 'familiestied',
             name: 'Families Tied',
             label: 'famtie',
             url: 'https://www.kink.com/channel/familiestied',
             description: 'Intense BDSM family role play threesomes & more.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        footworship: {
+        {
+            id: 'footworship',
             name: 'Foot Worship',
             label: 'footws',
             url: 'https://www.kink.com/channel/footworship',
             description: 'Satisfy Your Foot Fetish With The Kinkiest Foot Action. Enjoy Trampling, Foot Jobs, High Heels, And Pantyhose.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        fuckedandbound: {
+        {
+            id: 'fuckedandbound',
             name: 'Fucked and Bound',
             label: 'fbound',
             url: 'https://www.kink.com/channel/fuckedandbound',
             description: 'Extreme Anal, Rope Bondage, & Brutal Face Fucking.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        fuckingmachines: {
+        {
+            id: 'fuckingmachines',
             name: 'Fucking Machines',
             label: 'fmachi',
             url: 'https://www.kink.com/channel/fuckingmachines',
             description: 'Machines Fucking Squirting Pussies With Extreme Insertions. Fucking Machines is the ultimate hardcore sex toy porn. Huge dildos strapped to sex machines relentlessly fucking pornstars to real squirting oragsms!',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        hardcoregangbang: {
+        {
-            // discontinued
+            id: 'hardcoregangbang',
             name: 'Hardcore Gangbang',
             label: 'hardgb',
             url: 'https://www.kink.com/channel/hardcoregangbang',
-            description: 'Where all women\'s hardcore gangbang fantasies come true. Watch extreme, brutal gangbangs with pornstars, models, & MILFs that crave cock in every hole. HardcoreGangbang.com has the best creampie gang bangs online.',
+            description: "Where all women's hardcore gangbang fantasies come true. Watch extreme, brutal gangbangs with pornstars, models, & MILFs that crave cock in every hole. HardcoreGangbang.com has the best creampie gang bangs online.",
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        hogtied: {
+        {
+            id: 'hogtied',
             name: 'Hogtied',
             label: 'hogtie',
             url: 'https://www.kink.com/channel/hogtied',
             description: 'Your favorite girls restrained with rope, punished & trained. Hogtied is the original extreme bondage porn website. Watch top pornstars and pain sluts in brutal bondage, getting tormented, and forced to orgasm!',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        kinkuniversity: {
+        {
+            id: 'kinkuniversity',
             name: 'Kink University',
             label: 'kinkun',
             url: 'https://www.kink.com/channel/kinkuniversity',
             description: 'Learn BDSM Technical Skills & Theories From Respected Teachers In The Kink Community. Learn BDSM skills and improve your sex techniques. Video tutorials feature top sex ed experts and hardcore demos on topics from bondage to relationships.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        meninpain: {
+        {
+            id: 'meninpain',
             name: 'Men In Pain',
             label: 'meninp',
             url: 'https://www.kink.com/channel/meninpain',
             description: 'Submissive Men Violated With Verbal Humiliation And Harsh Punishment By Beautiful Dominatrices.',
-            tags: ['bdsm', 'femdom'],
+            network_id: 'kink',
         },
-        menonedge: {
+        {
+            id: 'menonedge',
             name: 'Men on Edge',
             label: 'moedge',
             url: 'https://www.kink.com/channel/menonedge',
-            description: 'Hot Guys Begging To Cum Are Brought To The Edge Of Complete Submission And Allowed To Blow Their Loads. Men on Edge has perfected the art of gay BDSM & edging porn. Watch straight men bound up & edged by dominant gay pornstars until they can\'t help but cum!',
+            description: "Hot Guys Begging To Cum Are Brought To The Edge Of Complete Submission And Allowed To Blow Their Loads. Men on Edge has perfected the art of gay BDSM & edging porn. Watch straight men bound up & edged by dominant gay pornstars until they can't help but cum!",
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        nakedkombat: {
+        {
+            id: 'nakedkombat',
             name: 'Naked Kombat',
             label: 'kombat',
             url: 'https://www.kink.com/channel/nakedkombat',
             description: 'Fight Fit Studs Go Head To Head In A Battle For Dominance. The Loser Gets Pinned And Punish Fucked Without Mercy',
-            tags: ['bdsm', 'gay'],
+            network_id: 'kink',
         },
-        publicdisgrace: {
+        {
+            id: 'publicdisgrace',
             name: 'Public Disgrace',
             label: 'pubdis',
             url: 'https://www.kink.com/channel/publicdisgrace',
             description: 'Women Bound Stripped And Punished In Public Get Hardcore Fucked Where Everyone Can See. Unscripted public humiliation & punishment of submissive slaves in real life locations. PublicDisgrace features the best outdoor BDSM & voyeur porn!',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        sadisticrope: {
+        {
+            id: 'sadisticrope',
             name: 'Sadistic Rope',
             label: 'sadisr',
             url: 'https://www.kink.com/channel/sadisticrope',
             description: 'Innocence Taken By Extreme Rope Bondage, Hardcore BDSM And Pussy-Destroying Orgasms.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        sexandsubmission: {
+        {
+            id: 'sexandsubmission',
             name: 'Sex and Submission',
             label: 'sexsub',
             url: 'https://www.kink.com/channel/sexandsubmission',
             description: 'Submissive Sluts Are Dominated With Rough Sex And Bondage. Real pornstars, hardcore bondage, master & slave roles are what SexAndSubmission.com is all about. Watch submissive sluts give in to total domination!',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        thetrainingofo: {
+        {
-            // discontinued
+            id: 'thetrainingofo',
             name: 'The Training of O',
             label: 'traino',
             url: 'https://www.kink.com/channel/thetrainingofo',
             description: 'Slaves Are Trained And Rewarded With Hardcore Bondage And Sex. Watch real pornstars undergo extreme slave training through hardcore bondage & BDSM porn. The Training of O is the ultimate slave / master experience!',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        theupperfloor: {
+        {
+            id: 'theupperfloor',
             name: 'The Upper Floor',
             label: 'upperf',
             url: 'https://www.kink.com/channel/theupperfloor',
             description: 'Trained slaves serve the house and their master in intense BDSM and kinky threesomes. The Upper Floor is a voyeuristic look into BDSM and fetish porn shoots with real submissive pornstars living out their kinky fantasies live on cam.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        tspussyhunts: {
+        {
+            id: 'tspussyhunts',
             name: 'TS Pussy Hunters',
             label: 'tspuss',
             url: 'https://www.kink.com/channel/tspussyhunters',
             description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.',
-            tags: ['bdsm', 'transsexual'],
+            network_id: 'kink',
         },
-        tsseduction: {
+        {
+            id: 'tsseduction',
             name: 'TS Seduction',
             label: 'tseduc',
             url: 'https://www.kink.com/channel/tsseduction',
             description: 'Sexy TS Women With Huge Cocks Dominate The Holes Of Straight Boys. Real TS women who are drop-dead gorgeous from their pretty faces to their big tits to their hard TS cocks. TS Seduction is the ultimate in transsexual bondage porn.',
-            tags: ['bdsm', 'transsexual'],
+            network_id: 'kink',
         },
-        ultimatesurrender: {
+        {
+            id: 'ultimatesurrender',
             name: 'Ultimate Surrender',
             label: 'ultsur',
             url: 'https://www.kink.com/channel/ultimatesurrender',
             description: 'Competitive Female Wrestling Where The Loser Gets Strap-On Punish Fucked. Ultimate Surrender features hardcore naked female wrestling porn videos where the winner gets to dominate the loser with some kinky lesbian FemDom!',
-            tags: ['bdsm', 'lesbian'],
+            network_id: 'kink',
         },
-        waterbondage: {
+        {
+            id: 'waterbondage',
             name: 'Water Bondage',
             label: 'waterb',
             url: 'https://www.kink.com/channel/waterbondage',
             description: 'Helpless Bound Beauties Sprayed, Dunked And Tormented Until They Cum Hard & Wet.',
-            tags: ['bdsm'],
+            network_id: 'kink',
         },
-        whippedass: {
+        {
+            id: 'whippedass',
             name: 'Whipped Ass',
             label: 'whippd',
             url: 'https://www.kink.com/channel/whippedass',
             description: 'Beautiful Submissive Sluts Take A Hard Fucking From Powerful Dominant Women. Watch brutal lesbian dominatrixes push submissive sluts to their orgasmic breaking points on WhippedAss! Hardcore fisting, huge strapons & face sitting!',
-            tags: ['bdsm', 'lesbian'],
+            network_id: 'kink',
         },
-        wiredpussy: {
+        {
+            id: 'wiredpussy',
             name: 'Wired Pussy',
             label: 'wiredp',
             url: 'https://www.kink.com/channel/wiredpussy',
             description: 'Gorgeous Women Submit To Electricity, Are Zapped, Shocked & Prodded To Orgasm.',
-            tags: ['bdsm', 'lesbian'],
+            network_id: 'kink',
         },
-    },
+        // LEGALPORNO
-};
+        {
+            name: 'LegalPorno',
+            label: 'legalp',
+            url: 'https://www.legalporno.com',
+            description: 'The Best HD Porn For You!',
+            network_id: 'legalporno',
+        },
+        // PERVCITY
+        {
+            id: 'analoverdose',
+            name: 'Anal Overdose',
+            label: 'AnalOD',
+            description: 'Before proceeding, use caution: the stunning pornstars of Anal Overdose are so fiery that they cause heavy breathing, throbbing cocks and volcanic loads of cum. If you think you can handle the heat of smoking tits, sweltering pussy and red hot ass.',
+            url: 'http://www.analoverdose.com',
+            network_id: 'pervcity',
+        },
+        {
+            id: 'bangingbeauties',
+            name: 'Banging Beauties',
+            label: 'BBeaus',
+            description: "Banging Beauties isn't just a porn site; it's the gateway to all your pussy-obsessed fantasies! Our members' area is flowing with beautiful pornstars anticipating big dick throbbing in their syrupy pink slits. These experienced babes love brutal vaginal pounding! Similarly, they're eager for anal switch-hitting to shake things up. However, it's not only about gorgeous sexperts filling their hungry holes. Sometimes, it's all about innocent rookies earning their pornstar status in first time threesomes and premier interracial scenes.",
+            url: 'http://www.bangingbeauties.com',
+            network_id: 'pervcity',
+        },
+        {
+            id: 'oraloverdose',
+            name: 'Oral Overdose',
+            label: 'OralOD',
+            description: "Oral Overdose is the only site you need to live out every saliva soaked blowjob of your dreams in HD POV! We've got the most stunning cocksuckers in the world going to town on big dick. These babes not only love cock, they can't get enough of it! In fact, there is no prick too huge for our hungry girls' throats. You'll find gorgeous, big tits pornstars exercising their gag reflex in intense balls deep facefuck scenes. We also feature fresh, young newbies taking on the gagging deepthroat challenge.",
+            url: 'http://www.oraloverdose.com',
+            network_id: 'pervcity',
+        },
+        {
+            id: 'chocolatebjs',
+            name: 'Chocolate BJs',
+            label: 'ChocBJ',
+            description: "You've just won the golden ticket to the best Chocolate BJs on the planet! We've sought far and wide to bring you the most beautiful black and ethnic pornstars. And they're in our members' area now! They can't wait to suck your white lollipop and lick the thick cream shooting from your big dick. Of course, no matter how sweet the booty or juicy the big tits, these brown foxes aren't all sugar and spice. In fact, when it comes to giving head, these big ass ebony babes know what they want: huge white cocks filling their throats!",
+            url: 'http://www.chocolatebjs.com',
+            network_id: 'pervcity',
+        },
+        {
+            id: 'upherasshole',
+            name: 'Up Her Asshole',
+            label: 'UpHerA',
+            description: "You don't need to travel the globe in search of the anal wonders of the world, because you get your own private tour right here on Up Her Asshole! Our stunning pornstars and rookie starlets welcome all ass fetish and anal sex fans, with their twerking bubble butts and winking assholes. However, big booty worship is just a slice of the fun. Combined with juicy tits (big and small), wet pussy (hairy and bald), these girls deliver a spectacular sensory experience in HD POV. Not only are you in danger of busting a nut before the going gets good, but also when the good turns remarkable with rimming, fingering and butt toys!",
+            url: 'http://www.upherasshole.com',
+            network_id: 'pervcity',
+        },
+        // XEMPIRE
+        {
+            id: 'hardx',
+            name: 'HardX',
+            label: 'HardX',
+            description: "Welcome to HardX.com, home of exclusive hardcore gonzo porn and first time anal scenes, DP, blowbangs and gangbangs from today's hottest porn stars!",
+            url: 'https://www.hardx.com',
+            network_id: 'xempire',
+        },
+        {
+            id: 'eroticax',
+            name: 'EroticaX',
+            label: 'EroX',
+            description: 'EroticaX.com features intimate scenes of passionate, erotic sex. Watch the sensual side of hardcore porn as your favorite pornstars have real, intense orgasms.',
+            url: 'https://www.eroticax.com',
+            network_id: 'xempire',
+        },
+        {
+            id: 'darkx',
+            name: 'DarkX',
+            label: 'DarkX',
+            description: 'Watch interracial BBC porn videos on DarkX.com, featuring the best pornstars taking big black cock in exclusive scenes. The best black on white porn inside!',
+            url: 'https://www.darkx.com',
+            network_id: 'xempire',
+        },
+        {
+            id: 'lesbianx',
+            name: 'LesbianX',
+            label: 'LesX',
+            description: "LesbianX.com features today's top pornstars in hardcore lesbian porn. Watch passionate & intense girl on girl sex videos, from erotic kissing to pussy licking.",
+            url: 'https://www.lesbianx.com',
+            network_id: 'xempire',
+        },
+    ]));

seeds/tags.js

@@ -0,0 +1,473 @@
+'use strict';
+
+/* eslint-disable max-len */
+exports.seed = knex => Promise.resolve()
+    .then(() => knex('tags').del())
+    .then(() => knex('tags').insert([
+        {
+            tag: 'airtight',
+            alias_for: null,
+        },
+        {
+            tag: 'anal',
+            alias_for: null,
+        },
+        {
+            tag: 'asian',
+            alias_for: null,
+        },
+        {
+            tag: 'ass licking',
+            alias_for: null,
+        },
+        {
+            tag: 'ATM',
+            alias_for: null,
+        },
+        {
+            tag: 'BDSM',
+            alias_for: null,
+        },
+        {
+            tag: 'BBC',
+            alias_for: null,
+        },
+        {
+            tag: 'big cock',
+            alias_for: null,
+        },
+        {
+            tag: 'big butt',
+            alias_for: null,
+        },
+        {
+            tag: 'big boobs',
+            alias_for: null,
+        },
+        {
+            tag: 'blonde',
+            alias_for: null,
+        },
+        {
+            tag: 'blowjob',
+            alias_for: null,
+        },
+        {
+            tag: 'blowbang',
+            alias_for: null,
+        },
+        {
+            tag: 'bondage',
+            alias_for: null,
+        },
+        {
+            tag: 'brunette',
+            alias_for: null,
+        },
+        {
+            tag: 'bukkake',
+            alias_for: null,
+        },
+        {
+            tag: 'cheerleader',
+            alias_for: null,
+        },
+        {
+            tag: 'choking',
+            alias_for: null,
+        },
+        {
+            tag: 'corporal punishment',
+            alias_for: null,
+        },
+        {
+            tag: 'creampie',
+            alias_for: null,
+        },
+        {
+            tag: 'cumshot',
+            alias_for: null,
+        },
+        {
+            tag: 'deepthroat',
+            alias_for: null,
+        },
+        {
+            tag: 'DAP',
+            alias_for: null,
+        },
+        {
+            tag: 'DP',
+            alias_for: null,
+        },
+        {
+            tag: 'DVP',
+            alias_for: null,
+        },
+        {
+            tag: 'double blowjob',
+            alias_for: null,
+        },
+        {
+            tag: 'ebony',
+            alias_for: null,
+        },
+        {
+            tag: 'facefucking',
+            alias_for: null,
+        },
+        {
+            tag: 'facial',
+            alias_for: null,
+        },
+        {
+            tag: 'facials',
+            alias_for: 'facial',
+        },
+        {
+            tag: 'gangbang',
+            alias_for: null,
+        },
+        {
+            tag: 'gaping',
+            alias_for: null,
+        },
+        {
+            tag: 'gonzo',
+            alias_for: null,
+        },
+        {
+            tag: 'hairy',
+            alias_for: null,
+        },
+        {
+            tag: 'hardcore',
+            alias_for: null,
+        },
+        {
+            tag: 'high heels',
+            alias_for: null,
+        },
+        {
+            tag: 'interracial',
+            alias_for: null,
+        },
+        {
+            tag: 'latina',
+            alias_for: null,
+        },
+        {
+            tag: 'lingerie',
+            alias_for: null,
+        },
+        {
+            tag: 'matag',
+            alias_for: null,
+        },
+        {
+            tag: 'MILF',
+            alias_for: null,
+        },
+        {
+            tag: 'natural',
+            alias_for: null,
+        },
+        {
+            tag: 'petite',
+            alias_for: null,
+        },
+        {
+            tag: 'pussy licking',
+            alias_for: null,
+        },
+        {
+            tag: 'redhead',
+            alias_for: null,
+        },
+        {
+            tag: 'roleplay',
+            alias_for: null,
+        },
+        {
+            tag: 'rough',
+            alias_for: null,
+        },
+        {
+            tag: 'schoolgirl',
+            alias_for: null,
+        },
+        {
+            tag: 'shaved',
+            alias_for: null,
+        },
+        {
+            tag: 'slapping',
+            alias_for: null,
+        },
+        {
+            tag: 'small boobs',
+            alias_for: null,
+        },
+        {
+            tag: 'squirting',
+            alias_for: null,
+        },
+        {
+            tag: 'swallowing',
+            alias_for: null,
+        },
+        {
+            tag: 'stockings',
+            alias_for: null,
+        },
+        {
+            tag: 'tattoo',
+            alias_for: null,
+        },
+        {
+            tag: 'threesome',
+            alias_for: null,
+        },
+        {
+            tag: 'teen',
+            alias_for: null,
+        },
+        {
+            tag: 'toy',
+            alias_for: null,
+        },
+        {
+            tag: 'TP',
+            alias_for: null,
+        },
+        {
+            tag: 'white',
+            alias_for: null,
+        },
+    ]))
+    .then(() => knex('tags').insert([
+        {
+            tag: '3+ on 1',
+            alias_for: 'gangbang',
+        },
+        {
+            tag: 'anilingus',
+            alias_for: 'ass licking',
+        },
+        {
+            tag: 'asians',
+            alias_for: 'asian',
+        },
+        {
+            tag: 'ass to mouth',
+            alias_for: 'ATM',
+        },
+        {
+            tag: 'atm',
+            alias_for: 'ATM',
+        },
+        {
+            tag: 'bbc',
+            alias_for: 'BBC',
+        },
+        {
+            tag: 'bdsm',
+            alias_for: 'BDSM',
+        },
+        {
+            tag: 'big ass',
+            alias_for: 'big butt',
+        },
+        {
+            tag: 'big black cock',
+            alias_for: 'BBC',
+        },
+        {
+            tag: 'big black cocks',
+            alias_for: 'BBC',
+        },
+        {
+            tag: 'big cocks',
+            alias_for: 'big cock',
+        },
+        {
+            tag: 'big butts',
+            alias_for: 'big butt',
+        },
+        {
+            tag: 'big tits',
+            alias_for: 'big boobs',
+        },
+        {
+            tag: 'black',
+            alias_for: 'ebony',
+        },
+        {
+            tag: 'blondes',
+            alias_for: 'blonde',
+        },
+        {
+            tag: 'blowjobs',
+            alias_for: 'blowjob',
+        },
+        {
+            tag: 'blowjob (double)',
+            alias_for: 'double blowjob',
+        },
+        {
+            tag: 'brunettes',
+            alias_for: 'brunette',
+        },
+        {
+            tag: 'cheer leader',
+            alias_for: 'cheerleader',
+        },
+        {
+            tag: 'creampies',
+            alias_for: 'creampie',
+        },
+        {
+            tag: 'cum swallowing',
+            alias_for: 'swallowing',
+        },
+        {
+            tag: 'cunnilingus',
+            alias_for: 'pussy licking',
+        },
+        {
+            tag: 'dap',
+            alias_for: 'DAP',
+        },
+        {
+            tag: 'deep throat',
+            alias_for: 'deepthroat',
+        },
+        {
+            tag: 'double anal penetration',
+            alias_for: 'DAP',
+        },
+        {
+            tag: 'double anal (dap)',
+            alias_for: 'DAP',
+        },
+        {
+            tag: 'double anal penetration (dap)',
+            alias_for: 'DAP',
+        },
+        {
+            tag: 'double penetration',
+            alias_for: 'DP',
+        },
+        {
+            tag: 'double penetration (dp)',
+            alias_for: 'DP',
+        },
+        {
+            tag: 'dp',
+            alias_for: 'DP',
+        },
+        {
+            tag: 'DPP',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double vaginal penetration',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double vaginal (dvp)',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double vaginal penetration (dvp)',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double vaginal (dpp)',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double pussy penetration',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'double pussy penetration (dpp)',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'dvp',
+            alias_for: 'DVP',
+        },
+        {
+            tag: 'gape',
+            alias_for: 'gaping',
+        },
+        {
+            tag: 'gapes',
+            alias_for: 'gaping',
+        },
+        {
+            tag: 'gapes (gaping asshole)',
+            alias_for: 'gaping',
+        },
+        {
+            tag: 'huge toys',
+            alias_for: 'toys',
+        },
+        {
+            tag: 'red head',
+            alias_for: 'redhead',
+        },
+        {
+            tag: 'milf',
+            alias_for: 'MILF',
+        },
+        {
+            tag: 'rimming',
+            alias_for: 'ass licking',
+        },
+        {
+            tag: 'rimjob',
+            alias_for: 'ass licking',
+        },
+        {
+            tag: 'role play',
+            alias_for: 'roleplay',
+        },
+        {
+            tag: 'rough sex',
+            alias_for: 'rough',
+        },
+        {
+            tag: 'school girl',
+            alias_for: 'schoolgirl',
+        },
+        {
+            tag: 'small tits',
+            alias_for: 'small boobs',
+        },
+        {
+            tag: 'swallow',
+            alias_for: 'swallowing',
+        },
+        {
+            tag: 'tattoos',
+            alias_for: 'tattoo',
+        },
+        {
+            tag: 'teens',
+            alias_for: 'teen',
+        },
+        {
+            tag: 'toys',
+            alias_for: 'toy',
+        },
+        {
+            tag: 'tp',
+            alias_for: 'TP',
+        },
+        {
+            tag: 'triple penetration',
+            alias_for: 'TP',
+        },
+    ]));

src/fetch-releases.js

@@ -3,86 +3,51 @@
 const config = require('config');
 const moment = require('moment');
 
-const networks = require('./networks');
+const knex = require('./knex');
 const scrapers = require('./scrapers');
 
-function accumulateIncludedSites() {
+function destructConfigNetworks(networks) {
-    return config.include.reduce((acc, network) => {
+    return networks.reduce((acc, network) => {
-        // network included with specific sites, only include specified sites
         if (Array.isArray(network)) {
-            const [networkId, siteIds] = network;
+            // network specifies sites
-
+            return {
-            return [
                 ...acc,
-                ...siteIds.map(siteId => ({
+                sites: [...acc.sites, ...network[1]],
-                    id: siteId,
+            };
-                    network: networkId,
-                    ...networks[networkId].sites[siteId],
-                })),
-            ];
         }
 
-        // network included without further specification, include all sites
+        return {
-        return [
             ...acc,
-            ...Object.entries(networks[network].sites).map(([siteId, site]) => ({
+            networks: [...acc.networks, network],
-                id: siteId,
+        };
-                network,
+    }, {
-                ...site,
+        networks: [],
-            })),
+        sites: [],
-        ];
+    });
-    }, []);
 }
 
-function accumulateExcludedSites() {
+function curateSites(sites) {
-    return Object.entries(networks).reduce((acc, [networkId, network]) => {
+    return sites.map(site => ({
-        const excludedNetwork = config.exclude.find((excludedNetworkX) => {
+        id: site.id,
-            if (Array.isArray(excludedNetworkX)) {
+        name: site.name,
-                return excludedNetworkX[0] === networkId;
+        description: site.description,
-            }
+        url: site.url,
-
+        networkId: site.network_id,
-            return excludedNetworkX === networkId;
+    }));
-        });
-
-        // network excluded with specific sites, only exclude specified sites
-        if (excludedNetwork && Array.isArray(excludedNetwork)) {
-            const [, excludedSiteIds] = excludedNetwork;
-
-            return [
-                ...acc,
-                ...Object.entries(network.sites)
-                    .filter(([siteId]) => !excludedSiteIds.includes(siteId))
-                    .map(([siteId, site]) => ({
-                        id: siteId,
-                        network: networkId,
-                        ...site,
-                    })),
-            ];
-        }
-
-        // network excluded without further specification, exclude all its sites
-        if (excludedNetwork) {
-            return acc;
-        }
-
-        // network not excluded, include all its sites
-        return [
-            ...acc,
-            ...Object.entries(network.sites).map(([siteId, site]) => ({
-                id: siteId,
-                network: networkId,
-                ...site,
-            })),
-        ];
-    }, []);
 }
 
-function accumulateSites() {
+async function accumulateIncludedSites() {
-    return config.include ? accumulateIncludedSites() : accumulateExcludedSites();
+    const included = destructConfigNetworks(config.include);
+
+    const rawSites = await knex('sites')
+        .whereIn('id', included.sites)
+        .orWhereIn('network_id', included.networks);
+
+    return curateSites(rawSites);
 }
 
 async function fetchReleases() {
-    const sites = await accumulateSites();
+    const sites = await accumulateIncludedSites();
 
     const scenesPerSite = await Promise.all(sites.map(async (site) => {
         const scraper = scrapers[site.id] || scrapers[site.network];

src/fetch-scene.js

@@ -3,30 +3,21 @@
 const config = require('config');
 const moment = require('moment');
 
-const networks = require('./networks');
+const knex = require('./knex');
 const scrapers = require('./scrapers');
 
-function findSite(url) {
+async function findSite(url) {
     const { origin } = new URL(url);
 
-    return Object.entries(networks)
+    const site = await knex('sites').where({ url: origin }).first();
-        .reduce((foundNetwork, [networkId, network]) => foundNetwork || Object.entries(network.sites)
-            .reduce((foundSite, [siteId, site]) => {
-                if (foundSite) return foundSite;
-                if (site.url !== origin) return null;
 
-                return {
+    return {
-                    site: {
+        id: site.id,
-                        ...site,
+        name: site.name,
-                        id: siteId,
+        description: site.description,
-                    },
+        url: site.url,
-                    network: {
+        networkId: site.network_id,
-                        ...network,
+    };
-                        id: networkId,
-                    },
-                };
-            }, null),
-        null);
 }
 
 function deriveFilename(scene) {
@@ -59,8 +50,8 @@ function deriveFilename(scene) {
 }
 
 async function fetchScene(url) {
-    const { site, network } = findSite(url);
+    const site = await findSite(url);
-    const scraper = scrapers[site.id] || scrapers[network.id];
+    const scraper = scrapers[site.id] || scrapers[site.networkId];
 
     if (!scraper) {
         throw new Error('Could not find scraper for URL');

src/knex.js

@@ -0,0 +1,11 @@
+'use strict';
+
+const knex = require('knex');
+
+module.exports = knex({
+    client: 'sqlite3',
+    connection: {
+        filename: './db.sqlite',
+    },
+    useNullAsDefault: true,
+});

src/networks/index.js

@@ -1,15 +0,0 @@
-'use strict';
-
-const julesjordan = require('./julesjordan');
-const kink = require('./kink');
-const legalporno = require('./legalporno');
-const pervcity = require('./pervcity');
-const xempire = require('./xempire');
-
-module.exports = {
-    julesjordan,
-    kink,
-    legalporno,
-    pervcity,
-    xempire,
-};

src/networks/julesjordan.js

@@ -1,36 +0,0 @@
-'use strict';
-
-module.exports = {
-    name: 'Jules Jordan',
-    url: 'https://www.julesjordan.com',
-    sites: {
-        julesjordan: {
-            name: 'Jules Jordan',
-            label: 'julesj',
-            url: 'https://www.julesjordan.com',
-            description: 'Jules Jordan\'s Official Membership Site',
-        },
-        /* also listed on main site
-        manuel: {
-            name: 'Manuel Ferrara',
-            label: 'manuel',
-            url: 'http://www.manuelferrara.com/trial',
-            description: 'Manuel Ferrara\'s Official Site',
-        },
-        */
-        /* no listed date
-        theassfactory: {
-            name: 'The Ass Factory',
-            label: 'assfac',
-            url: 'https://www.theassfactory.com/trial',
-        },
-        */
-        /* no updates since 2016
-        spermswallowers: {
-            name: 'Sperm Swallowers',
-            label: 'sperms',
-            url: 'https://www.spermswallowers.com/trial',
-        },
-        */
-    },
-};

src/networks/legalporno.js

@@ -1,16 +0,0 @@
-'use stric';
-
-module.exports = {
-    name: 'LegalPorno',
-    url: 'https://www.legalporno.com',
-    description: 'The Best HD Porn For You!',
-    sites: {
-        legalporno: {
-            name: 'LegalPorno',
-            label: 'legalp',
-            url: 'https://www.legalporno.com',
-            description: 'The Best HD Porn For You!',
-            tags: ['gangbang'],
-        },
-    },
-};

src/networks/pervcity.js

@@ -1,59 +0,0 @@
-'use strict';
-
-/* eslint-disable max-len */
-module.exports = {
-    name: 'Perv City',
-    url: 'https://www.pervcity.com',
-    description: '',
-    sites: {
-        analoverdose: {
-            name: 'Anal Overdose',
-            label: 'AnalOD',
-            url: 'http://www.analoverdose.com',
-            description: 'Before proceeding, use caution: the stunning pornstars of Anal Overdose are so fiery that they cause heavy breathing, throbbing cocks and volcanic loads of cum. If you think you can handle the heat of smoking tits, sweltering pussy and red hot ass.',
-            tags: ['anal'],
-            parameters: {
-                tourId: 3,
-            },
-        },
-        bangingbeauties: {
-            name: 'Banging Beauties',
-            label: 'BBeaus',
-            url: 'http://www.bangingbeauties.com',
-            description: 'Banging Beauties isn\'t just a porn site; it\'s the gateway to all your pussy-obsessed fantasies! Our members\' area is flowing with beautiful pornstars anticipating big dick throbbing in their syrupy pink slits. These experienced babes love brutal vaginal pounding! Similarly, they\'re eager for anal switch-hitting to shake things up. However, it\'s not only about gorgeous sexperts filling their hungry holes. Sometimes, it\'s all about innocent rookies earning their pornstar status in first time threesomes and premier interracial scenes.',
-            parameters: {
-                tourId: 7,
-            },
-        },
-        oraloverdose: {
-            name: 'Oral Overdose',
-            label: 'OralOD',
-            url: 'http://www.oraloverdose.com',
-            description: 'Oral Overdose is the only site you need to live out every saliva soaked blowjob of your dreams in HD POV! We\'ve got the most stunning cocksuckers in the world going to town on big dick. These babes not only love cock, they can\'t get enough of it! In fact, there is no prick too huge for our hungry girls\' throats. You\'ll find gorgeous, big tits pornstars exercising their gag reflex in intense balls deep facefuck scenes. We also feature fresh, young newbies taking on the gagging deepthroat challenge.',
-            tags: ['blowjob'],
-            parameters: {
-                tourId: 4,
-            },
-        },
-        chocolatebjs: {
-            name: 'Chocolate BJs',
-            label: 'ChocBJ',
-            url: 'http://www.chocolatebjs.com',
-            description: 'You\'ve just won the golden ticket to the best Chocolate BJs on the planet! We\'ve sought far and wide to bring you the most beautiful black and ethnic pornstars. And they\'re in our members\' area now! They can\'t wait to suck your white lollipop and lick the thick cream shooting from your big dick. Of course, no matter how sweet the booty or juicy the big tits, these brown foxes aren\'t all sugar and spice. In fact, when it comes to giving head, these big ass ebony babes know what they want: huge white cocks filling their throats!',
-            tags: ['blowjob', 'interracial'],
-            parameters: {
-                tourId: 6,
-            },
-        },
-        upherasshole: {
-            name: 'Up Her Asshole',
-            label: 'UpHerA',
-            url: 'http://www.upherasshole.com',
-            description: 'You don\'t need to travel the globe in search of the anal wonders of the world, because you get your own private tour right here on Up Her Asshole! Our stunning pornstars and rookie starlets welcome all ass fetish and anal sex fans, with their twerking bubble butts and winking assholes. However, big booty worship is just a slice of the fun. Combined with juicy tits (big and small), wet pussy (hairy and bald), these girls deliver a spectacular sensory experience in HD POV. Not only are you in danger of busting a nut before the going gets good, but also when the good turns remarkable with rimming, fingering and butt toys!',
-            tags: ['anal'],
-            parameters: {
-                tourId: 9,
-            },
-        },
-    },
-};

src/networks/xempire.js

@@ -1,36 +0,0 @@
-'use strict';
-
-module.exports = {
-    name: 'XEmpire',
-    url: 'https://www.xempire.com',
-    description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
-    sites: {
-        hardx: {
-            name: 'HardX',
-            label: 'HardX',
-            url: 'https://www.hardx.com',
-            description: 'Welcome to HardX.com, home of exclusive hardcore gonzo porn and first time anal scenes, DP, blowbangs and gangbangs from today\'s hottest porn stars!',
-        },
-        eroticax: {
-            name: 'EroticaX',
-            label: 'EroX',
-            url: 'https://www.eroticax.com',
-            description: 'EroticaX.com features intimate scenes of passionate, erotic sex. Watch the sensual side of hardcore porn as your favorite pornstars have real, intense orgasms.',
-            tags: ['erotic'],
-        },
-        darkx: {
-            name: 'DarkX',
-            label: 'DarkX',
-            url: 'https://www.darkx.com',
-            description: 'Watch interracial BBC porn videos on DarkX.com, featuring the best pornstars taking big black cock in exclusive scenes. The best black on white porn inside!',
-            tags: ['interracial'],
-        },
-        lesbianx: {
-            name: 'LesbianX',
-            label: 'LesX',
-            url: 'https://www.lesbianx.com',
-            description: 'LesbianX.com features today\'s top pornstars in hardcore lesbian porn. Watch passionate & intense girl on girl sex videos, from erotic kissing to pussy licking.',
-            tags: ['lesbian'],
-        },
-    },
-};

src/scrapers/julesjordan.js

@@ -4,26 +4,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const tagMap = {
+const { matchTags } = require('../tags');
-    Anal: 'anal',
-    Asian: 'asian',
-    'Ass To Mouth': 'ATM',
-    'Big Cocks': 'big cock',
-    Black: 'BBC',
-    Blondes: 'blonde',
-    Brunettes: 'brunette',
-    Blowjobs: 'blowjob',
-    Creampie: 'creampie',
-    'Deep Throat': 'deepthroat',
-    Facial: 'facial',
-    Interracial: 'interracial',
-    Lingerie: 'lingerie',
-    Natural: 'natural',
-    'Red Head': 'readhead',
-    'School Girl': 'schoolgirl',
-    Tattoo: 'tattoo',
-    Teen: 'teen',
-};
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -87,7 +68,7 @@ function scrapeUpcoming(html, site) {
     });
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
     const title = $('.title_bar_hilite').text();
@@ -104,7 +85,7 @@ function scrapeScene(html, url, site) {
     const stars = Number($('.avg_rating').text().trim().replace(/[\s|Avg Rating:]/g, ''));
 
     const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,
@@ -120,7 +101,7 @@ function scrapeScene(html, url, site) {
     };
 }
 
-async function fetchLatest(site) {
+async function fetchLatest(site, _date) {
     const res = await bhttp.get(`${site.url}/trial/categories/movies_1_d.html`);
 
     return scrapeLatest(res.body.toString(), site);

src/scrapers/kink.js

@@ -3,31 +3,9 @@
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
-const { kink } = require('../networks');
 
-const tagMap = {
+const knex = require('../knex');
-    Airtight: 'airtight',
+const { matchTags } = require('../tags');
-    Anal: 'anal',
-    Asian: 'asian',
-    BDSM: 'BDSM',
-    Blowbang: 'blowbang',
-    Blowjob: 'blowjob',
-    Bondage: 'bondage',
-    Choking: 'choking',
-    'Corporal Punishment': 'corporal punishment',
-    'Double Penetration': 'DP',
-    Gangbang: 'gangbang',
-    'High Heels': 'high heels',
-    Petite: 'petite',
-    'Role Play': 'roleplay',
-    'Rope Bondage': 'bondage',
-    'Rough Sex': 'rough',
-    Shaved: 'shaved',
-    Slapping: 'slapping',
-    'Small Tits': 'small boobs',
-    Squirting: 'squirting',
-    White: 'white',
-};
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -71,7 +49,6 @@ async function scrapeScene(html, url, id, ratingRes, site) {
     const actorsRaw = $('.shoot-info p.starring');
 
     const sitename = $('.shoot-logo a').attr('href').split('/')[2];
-    const channelSite = kink.sites[sitename];
 
     const date = moment.utc($(actorsRaw)
         .prev()
@@ -87,7 +64,9 @@ async function scrapeScene(html, url, id, ratingRes, site) {
     const { average: stars } = ratingRes.body;
 
     const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+
+    const channelSite = await knex('sites').where({ id: sitename }).first();
+    const tags = await matchTags(rawTags);
 
     return {
         url,

src/scrapers/legalporno.js

@@ -4,6 +4,8 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
+const { matchTags } = require('../tags');
+
 const tagMap = {
     '3+ on 1': 'gangbang',
     anal: 'anal',
@@ -52,7 +54,7 @@ function scrapeLatest(html, site) {
     });
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
     const originalTitle = $('h1.watchpage-title').text().trim();
@@ -68,7 +70,7 @@ function scrapeScene(html, url, site) {
     const duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
 
     const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,

src/scrapers/xempire.js

@@ -4,35 +4,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const tagMap = {
+const { matchTags } = require('../tags');
-    Anal: 'anal',
-    'Ass Licking': 'ass licking',
-    'Ass To Mouth': 'ATM',
-    'Big Ass': 'big butt',
-    'Big Tits': 'big boobs',
-    Black: 'big black cock',
-    Blonde: 'blonde',
-    Blowjob: 'blowjob',
-    'Blowjob (double)': 'double blowjob',
-    Brunette: 'brunette',
-    'Cum Swallowing': 'swallowing',
-    Cumshot: 'cumshot',
-    Deepthroat: 'deepthroat',
-    'Double Penetration (DP)': 'DP',
-    Ebony: 'ebony',
-    Facial: 'facial',
-    Gangbang: 'gangbang',
-    Gonzo: 'gonzo',
-    Hardcore: 'hardcore',
-    Interracial: 'interracial',
-    Latina: 'latina',
-    Petite: 'petite',
-    'Pussy Licking': 'pussy licking',
-    Rimjob: 'ass licking',
-    'Rough Sex': 'rough',
-    'Small Tits': 'small boobs',
-    Threesome: 'threesome',
-};
 
 function scrape(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -69,7 +41,7 @@ function scrape(html, site) {
     });
 }
 
-function scrapeSceneFallback($, url, site) {
+async function scrapeSceneFallback($, url, site) {
     const title = $('h1.title').text();
     const date = moment.utc($('.updatedDate').text(), 'MM-DD-YYYY').toDate();
     const actors = $('.sceneColActors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
@@ -78,7 +50,7 @@ function scrapeSceneFallback($, url, site) {
     const stars = $('.currentRating').text().split('/')[0] / 2;
 
     const rawTags = $('.sceneColCategories > a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,
@@ -94,7 +66,7 @@ function scrapeSceneFallback($, url, site) {
     };
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
     const json = $('script[type="application/ld+json"]').html();
 
@@ -122,7 +94,7 @@ function scrapeScene(html, url, site) {
     const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
 
     const rawTags = data.keywords.split(', ');
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,

src/tags.js

@@ -1,61 +1,14 @@
 'use strict';
 
-module.exports = {
+const knex = require('./knex');
-    airtight: [],
+
-    anal: [],
+async function matchTags(rawTags) {
-    asian: ['asians'],
+    const tagEntries = await knex('tags')
-    'ass licking': ['rimming', 'rimjob', 'anilingus'],
+        .select(knex.raw('ifnull(original.tag, tags.tag) as tag'), knex.raw('ifnull(original.tag, tags.tag) as tag'))
-    ATM: ['ass to mouth'],
+        .whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase()))
-    BDSM: [],
+        .leftJoin('tags as original', 'tags.alias_for', 'original.tag');
-    'big black cock': ['BBC', 'bbc', 'big black cock', 'big black cocks'],
+
-    'big cock': ['big cocks'],
+    return tagEntries.map(({ tag }) => tag);
-    'big butt': ['big butts'],
+}
-    'big boobs': ['big tits'],
+
-    blonde: ['blondes'],
+module.exports = { matchTags };
-    blowjob: [],
-    blowbang: [],
-    bondage: [],
-    brunette: ['brunettes'],
-    bukkake: [],
-    cheerleader: ['cheer leader'],
-    choking: [],
-    'corporal punishment': [],
-    creampie: ['creampies'],
-    cumshot: [],
-    deepthroat: ['deep throat'],
-    DAP: ['dap', 'double anal penetration'],
-    DP: ['dp', 'double penetration'],
-    DVP: ['DPP', 'dpp', 'dvp', 'double vaginal penetration', 'double pussy penetration'],
-    'double blowjob': [],
-    ebony: ['black'],
-    facefucking: [],
-    facial: ['facials'],
-    gangbang: [],
-    gaping: ['gape', 'gapes'],
-    gonzo: [],
-    hairy: [],
-    hardcore: [],
-    'high heels': [],
-    latina: [],
-    lingerie: [],
-    maid: [],
-    MILF: ['milf'],
-    petite: [],
-    'pussy licking': ['cunnilingus'],
-    redhead: ['red head'],
-    roleplay: ['role play'],
-    rough: [],
-    schoolgirl: ['school girl'],
-    shaved: [],
-    slapping: [],
-    'small boobs': ['small tits'],
-    squirting: [],
-    swallowing: ['swallow'],
-    stockings: [],
-    tattoo: ['tattoos'],
-    threesome: ['threesome'],
-    teen: ['teens'],
-    toy: ['toys'],
-    TP: ['tp', 'triple penetration'],
-    white: [],
-};