Saving results to database. Showing webpage.

assets/views/home.jsx

@@ -0,0 +1,34 @@
+'use strict';
+
+const React = require('react');
+const moment = require('moment');
+
+class Home extends React.Component {
+    render() {
+        return (
+            <table>
+                <tr>
+                    <th>Date</th>
+                    <th>ID</th>
+                    <th>Shoot ID / Entry ID</th>
+                    <th>Site</th>
+                    <th>Title</th>
+                    <th>Actors</th>
+                </tr>
+
+                {this.props.releases.map(release => (
+                    <tr key={release.id}>
+                        <td>{ moment(release.date).format('YYYY-MM-DD') }</td>
+                        <td>{ release.id }</td>
+                        <td>{ release.shootId || release.entryId }</td>
+                        <td>{ release.site.name }</td>
+                        <td>{ release.title }</td>
+                        <td>{ release.actors && release.actors.map(actor => actor.name).join(', ') }</td>
+                    </tr>
+                ))}
+            </table>
+        );
+    }
+}
+
+module.exports = Home;

config/default.js

@@ -1,6 +1,15 @@
 'use strict';
 
 module.exports = {
+    database: {
+        host: '127.0.0.1',
+        user: 'user',
+        password: 'password',
+        database: 'traxxx',
+    },
+    web: {
+        port: 5000,
+    },
     include: [
         '21sextury',
         'blowpass',

knexfile.js

@@ -1,9 +1,8 @@
-// Update with your config settings.
+// use strict
+//
+const config = require('config');
 
 module.exports = {
-    client: 'sqlite3',
+    client: 'pg',
-    connection: {
+    connection: config.database,
-        filename: './db.sqlite',
-    },
-    useNullAsDefault: true,
 };

migrations/20190325001339_releases.js

@@ -4,7 +4,9 @@ exports.up = knex => Promise.resolve()
     .then(() => knex.schema.createTable('actors', (table) => {
         table.increments('id', 8);
 
-        table.string('name');
+        table.string('name')
+            .unique()
+            .notNullable();
         table.string('gender', 18);
 
         table.integer('alias_for', 8)
@@ -23,17 +25,20 @@ exports.up = knex => Promise.resolve()
         table.string('group', 20)
             .primary();
 
-        table.string('name', 20);
+        table.string('name', 32);
     }))
     .then(() => knex.schema.createTable('tags', (table) => {
-        table.string('tag', 20)
+        table.string('tag', 32)
             .primary();
 
+        table.integer('capitalization', 1)
+            .defaultTo(0);
+
         table.string('group_id', 20)
             .references('group')
             .inTable('tags_groups');
 
-        table.string('alias_for', 20)
+        table.string('alias_for', 32)
             .references('tag')
             .inTable('tags');
     }))
@@ -43,14 +48,12 @@ exports.up = knex => Promise.resolve()
 
         table.string('name');
         table.string('url');
-        table.string('description');
+        table.text('description');
     }))
     .then(() => knex.schema.createTable('sites', (table) => {
         table.string('id', 32)
             .primary();
 
-        table.string('label', 6);
-
         table.string('network_id', 32)
             .notNullable()
             .references('id')
@@ -58,7 +61,7 @@ exports.up = knex => Promise.resolve()
 
         table.string('name');
         table.string('url');
-        table.string('description');
+        table.text('description');
         table.string('parameters');
     }))
     .then(() => knex.schema.createTable('releases', (table) => {
@@ -94,6 +97,9 @@ exports.up = knex => Promise.resolve()
 
         table.integer('rating')
             .unsigned();
+
+        table.datetime('created_at')
+            .defaultTo(knex.fn.now());
     }))
     .then(() => knex.schema.createTable('actors_associated', (table) => {
         table.increments('id', 16);
@@ -118,7 +124,7 @@ exports.up = knex => Promise.resolve()
             .references('id')
             .inTable('sites');
 
-        table.string('release_id')
+        table.integer('release_id', 12)
             .references('id')
             .inTable('releases');
     }));
@@ -126,10 +132,10 @@ exports.up = knex => Promise.resolve()
 exports.down = knex => Promise.resolve()
     .then(() => knex.schema.dropTable('tags_associated'))
     .then(() => knex.schema.dropTable('actors_associated'))
+    .then(() => knex.schema.dropTable('tags'))
+    .then(() => knex.schema.dropTable('tags_groups'))
+    .then(() => knex.schema.dropTable('actors'))
     .then(() => knex.schema.dropTable('releases'))
     .then(() => knex.schema.dropTable('sites'))
-    .then(() => knex.schema.dropTable('networks'))
-    .then(() => knex.schema.dropTable('actors'))
     .then(() => knex.schema.dropTable('directors'))
-    .then(() => knex.schema.dropTable('tags_groups'))
+    .then(() => knex.schema.dropTable('networks'));
-    .then(() => knex.schema.dropTable('tags'));

package.json

@@ -44,11 +44,18 @@
         "cheerio": "^1.0.0-rc.2",
         "clipboardy": "^1.2.3",
         "config": "^3.0.1",
+        "express": "^4.16.4",
+        "express-promise-router": "^3.0.3",
+        "express-react-views": "^0.11.0",
+        "fs-extra": "^7.0.1",
         "knex": "^0.16.3",
         "knex-migrate": "^1.7.1",
         "moment": "^2.24.0",
         "neo-blessed": "^0.2.0",
         "opn": "^5.4.0",
+        "pg": "^7.9.0",
+        "react": "^16.8.6",
+        "react-dom": "^16.8.6",
         "sqlite3": "^4.0.6",
         "tough-cookie": "^3.0.1",
         "tty-table": "^2.7.0",

seeds/00_networks.js

@@ -2,8 +2,7 @@
 
 /* eslint-disable max-len */
 exports.seed = knex => Promise.resolve()
-    .then(() => knex('networks').del())
+    .then(() => knex.raw(`${knex('networks').insert([
-    .then(() => knex('networks').insert([
         {
             id: '21sextury',
             name: '21Sextury',
@@ -69,6 +68,12 @@ exports.seed = knex => Promise.resolve()
             url: 'https://www.pervcity.com',
             description: '',
         },
+        {
+            id: 'pornpros',
+            name: 'Porn Pros',
+            url: 'https://pornpros.com',
+            description: 'Watch the best HD exclusive movies and videos on Porn Pros. All the hottest new Pornstar and amateur girls in High Definition updated daily.',
+        },
         {
             id: 'private',
             name: 'Private',
@@ -93,4 +98,4 @@ exports.seed = knex => Promise.resolve()
             url: 'https://www.xempire.com',
             description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
         },
-    ]));
+    ]).toString()} ON CONFLICT DO NOTHING`));

seeds/02_tags.js

@@ -2,9 +2,7 @@
 
 /* eslint-disable max-len */
 exports.seed = knex => Promise.resolve()
-    .then(() => knex('tags').del())
+    .then(() => knex.raw(`${knex('tags_groups').insert([
-    .then(() => knex('tags_groups').del())
-    .then(() => knex('tags_groups').insert([
         {
             group: 'age',
             name: 'Age',
@@ -45,8 +43,8 @@ exports.seed = knex => Promise.resolve()
             group: 'roleplay',
             name: 'Roleplay',
         },
-    ]))
+    ]).toString()} ON CONFLICT DO NOTHING`))
-    .then(() => knex('tags').insert([
+    .then(() => knex.raw(`${knex('tags').insert([
         {
             tag: '69',
             alias_for: null,
@@ -62,9 +60,10 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'American',
+            tag: 'american',
             alias_for: null,
             group_id: 'ethnicity',
+            capitalization: 1,
         },
         {
             tag: 'anal creampie',
@@ -105,8 +104,9 @@ exports.seed = knex => Promise.resolve()
             group_id: 'body',
         },
         {
-            tag: 'ATM',
+            tag: 'atm',
             alias_for: null,
+            capitalization: 2,
         },
         {
             tag: 'ball licking',
@@ -123,13 +123,15 @@ exports.seed = knex => Promise.resolve()
             group_id: 'location',
         },
         {
-            tag: 'BDSM',
+            tag: 'bdsm',
             alias_for: null,
+            capitalization: 2,
         },
         {
-            tag: 'BBC',
+            tag: 'bbc',
             alias_for: null,
             group_id: 'body',
+            capitalization: 2,
         },
         {
             tag: 'big cock',
@@ -224,24 +226,27 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'DAP',
+            tag: 'dap',
             alias_for: null,
+            capitalization: 2,
         },
         {
             tag: 'deepthroat',
             alias_for: null,
         },
         {
-            tag: 'DP',
+            tag: 'dp',
             alias_for: null,
+            capitalization: 2,
         },
         {
             tag: 'dungeon',
             alias_for: null,
         },
         {
-            tag: 'DVP',
+            tag: 'dvp',
             alias_for: null,
+            capitalization: 2,
         },
         {
             tag: 'double blowjob',
@@ -270,9 +275,10 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'European',
+            tag: 'european',
             alias_for: null,
             group_id: 'ethnicity',
+            capitalization: 1,
         },
         {
             tag: 'facefuck',
@@ -305,9 +311,10 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'FMF',
+            tag: 'fmf',
             alias_for: null,
             group_id: 'group',
+            capitalization: 2,
         },
         {
             tag: 'gag',
@@ -341,9 +348,10 @@ exports.seed = knex => Promise.resolve()
             group_id: 'clothing',
         },
         {
-            tag: 'Hungarian',
+            tag: 'hungarian',
             alias_for: null,
             group_id: 'ethnicity',
+            capitalization: 1,
         },
         {
             tag: 'humiliation',
@@ -396,14 +404,16 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'MILF',
+            tag: 'milf',
             alias_for: null,
             group_id: 'age',
+            capitalization: 2,
         },
         {
-            tag: 'MFM',
+            tag: 'mfm',
             alias_for: null,
             group_id: 'group',
+            capitalization: 2,
         },
         {
             tag: 'miniskirt',
@@ -484,9 +494,10 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'Russian',
+            tag: 'russian',
             alias_for: null,
             group_id: 'ethnicity',
+            capitalization: 1,
         },
         {
             tag: 'saliva',
@@ -594,8 +605,9 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
         },
         {
-            tag: 'TP',
+            tag: 'tp',
             alias_for: null,
+            capitalization: 2,
         },
         {
             tag: 'trimmed',
@@ -631,12 +643,12 @@ exports.seed = knex => Promise.resolve()
             alias_for: null,
             group_id: 'location',
         },
-    ]))
+    ]).toString()} ON CONFLICT DO NOTHING`))
-    .then(() => knex('tags').insert([
+    .then(() => knex.raw(`${knex('tags').insert([
         // ALIASES
         {
             tag: '2-on-1',
-            alias_for: 'MFM',
+            alias_for: 'mfm',
         },
         {
             tag: '3+ on 1',
@@ -656,7 +668,7 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'ass to mouth',
-            alias_for: 'ATM',
+            alias_for: 'atm',
         },
         {
             tag: 'bald pussy',
@@ -668,7 +680,7 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'BGB',
-            alias_for: 'MFM',
+            alias_for: 'mfm',
         },
         {
             tag: 'big ass',
@@ -676,11 +688,11 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'big black cock',
-            alias_for: 'BBC',
+            alias_for: 'bbc',
         },
         {
             tag: 'big black cocks',
-            alias_for: 'BBC',
+            alias_for: 'bbc',
         },
         {
             tag: 'big cocks',
@@ -840,11 +852,11 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'dom',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'domination',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'dominatrix',
@@ -852,51 +864,51 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'double anal penetration',
-            alias_for: 'DAP',
+            alias_for: 'dap',
         },
         {
             tag: 'double anal (dap)',
-            alias_for: 'DAP',
+            alias_for: 'dap',
         },
         {
             tag: 'double anal penetration (dap)',
-            alias_for: 'DAP',
+            alias_for: 'dap',
         },
         {
             tag: 'double penetration',
-            alias_for: 'DP',
+            alias_for: 'dp',
         },
         {
             tag: 'double penetration (dp)',
-            alias_for: 'DP',
+            alias_for: 'dp',
         },
         {
-            tag: 'DPP',
+            tag: 'dpP',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double vaginal penetration',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double vaginal (dvp)',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double vaginal penetration (dvp)',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double vaginal (dpp)',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double pussy penetration',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'double pussy penetration (dpp)',
-            alias_for: 'DVP',
+            alias_for: 'dvp',
         },
         {
             tag: 'drool',
@@ -926,6 +938,10 @@ exports.seed = knex => Promise.resolve()
             tag: 'face sitting',
             alias_for: 'facesitting',
         },
+        {
+            tag: 'facial cumshot',
+            alias_for: 'facial',
+        },
         {
             tag: 'facials',
             alias_for: 'facial',
@@ -995,12 +1011,12 @@ exports.seed = knex => Promise.resolve()
             alias_for: 'miniskirt',
         },
         {
-            tag: 'MMF',
+            tag: 'mmf',
-            alias_for: 'MFM',
+            alias_for: 'mfm',
         },
         {
-            tag: 'MFF',
+            tag: 'mff',
-            alias_for: 'FMF',
+            alias_for: 'fmf',
         },
         {
             tag: 'natural',
@@ -1076,11 +1092,11 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'sadomasochism',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'sadism',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'scissoring',
@@ -1100,7 +1116,7 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'slave',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'small ass',
@@ -1116,7 +1132,7 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'spitroast',
-            alias_for: 'MFM',
+            alias_for: 'mfm',
         },
         {
             tag: 'standing doggystyle',
@@ -1140,11 +1156,11 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'sub',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'submission',
-            alias_for: 'BDSM',
+            alias_for: 'bdsm',
         },
         {
             tag: 'tattoos',
@@ -1172,7 +1188,7 @@ exports.seed = knex => Promise.resolve()
         },
         {
             tag: 'triple penetration',
-            alias_for: 'TP',
+            alias_for: 'tp',
         },
         {
             tag: 'whipping',
@@ -1190,4 +1206,4 @@ exports.seed = knex => Promise.resolve()
             tag: 'zapper',
             alias_for: 'electric shock',
         },
-    ]));
+    ]).toString()} ON CONFLICT DO NOTHING`));

src/app.js

@@ -5,6 +5,7 @@ const clipboard = require('clipboardy');
 
 const argv = require('./argv');
 const { renderReleases, renderScene } = require('./tui/render');
+const initServer = require('./web/server');
 
 const fetchReleases = require('./fetch-releases');
 const fetchScene = require('./fetch-scene');
@@ -34,35 +35,43 @@ function getMethod() {
         };
     }
 
-    return {
+    if (argv.fetch) {
-        fetch: () => fetchReleases(),
+        return {
-        render: renderReleases,
+            fetch: () => fetchReleases(),
-    };
+            render: renderReleases,
+        };
+    }
+
+    return null;
 }
 
 async function init() {
-    const screen = argv.render && !argv.filename && initScreen();
+    initServer();
 
+    const screen = argv.render && !argv.filename && initScreen();
 
     try {
         const method = getMethod();
-        const result = await method.fetch();
 
-        if (result) {
+        if (method) {
-            if (argv.copy && result.copy) {
+            const result = await method.fetch();
-                clipboard.writeSync(result.copy);
-                console.log(`Result copied to clipboard: ${result.copy}`);
-            }
 
-            if (argv.filename && result.filename) {
+            if (result) {
-                console.log(result.filename);
+                if (argv.copy && result.copy) {
+                    clipboard.writeSync(result.copy);
+                    console.log(`Result copied to clipboard: ${result.copy}`);
+                }
 
-                // setTimeout(() => log(), 5000);
+                if (argv.filename && result.filename) {
-                return;
+                    console.log(result.filename);
-            }
 
-            if (argv.render) {
+                    // setTimeout(() => log(), 5000);
-                method.render(result, screen);
+                    return;
+                }
+
+                if (argv.render) {
+                    method.render(result, screen);
+                }
             }
         }
     } catch (error) {

src/argv.js

@@ -5,6 +5,16 @@ const yargs = require('yargs');
 
 const { argv } = yargs
     .command('npm start')
+    .option('fetch', {
+        describe: 'Fetch latest releases',
+        type: 'boolean',
+        default: false,
+    })
+    .option('deep', {
+        describe: 'Fetch details for all releases',
+        type: 'boolean',
+        default: false,
+    })
     .option('networks', {
         describe: 'Networks to include (overrides config)',
         type: 'array',
@@ -23,12 +33,12 @@ const { argv } = yargs
     .option('save', {
         describe: 'Save fetched releases to database',
         type: 'boolean',
-        default: true,
+        default: false,
     })
     .option('render', {
         describe: 'Fetch data without rendering interface',
         type: 'boolean',
-        default: true,
+        default: false,
     })
     .option('scene', {
         describe: 'Fetch scene info from URL',

src/fetch-releases.js

@@ -1,11 +1,13 @@
 'use strict';
 
 const config = require('config');
+const Promise = require('bluebird');
 const moment = require('moment');
 
 const argv = require('./argv');
 const knex = require('./knex');
 const scrapers = require('./scrapers');
+const fetchScene = require('./fetch-scene');
 
 function destructConfigNetworks(networks) {
     return networks.reduce((acc, network) => {
@@ -72,32 +74,48 @@ async function findDuplicateReleases(latestReleases, _siteId) {
         .orWhereIn('entry_id', latestReleasesEntryIds);
 }
 
-async function storeReleases(releases) {
+async function storeReleases(releases = []) {
-    const curatedReleases = releases.map(release => ({
+    return Promise.reduce(releases, async (acc, release) => {
-        site_id: release.site.id,
+        await acc;
-        shoot_id: release.shootId || null,
-        entry_id: release.entryId || null,
-        url: release.url,
-        title: release.title,
-        date: release.date,
-        description: release.description,
-        director: release.director,
-        duration: release.duration,
-        likes: release.rating && release.rating.likes,
-        dislikes: release.rating && release.rating.dislikes,
-        rating: release.rating && release.rating.stars,
-    }));
 
-    if (curatedReleases.length) {
+        const curatedRelease = {
-        console.log(`Saving ${curatedReleases.length} new releases to database`);
+            site_id: release.site.id,
+            shoot_id: release.shootId || null,
+            entry_id: release.entryId || null,
+            url: release.url,
+            title: release.title,
+            date: release.date,
+            description: release.description,
+            director: release.director,
+            duration: release.duration,
+            likes: release.rating && release.rating.likes,
+            dislikes: release.rating && release.rating.dislikes,
+            rating: release.rating && release.rating.stars,
+        };
 
-        const insertQuery = knex('releases').insert(curatedReleases).toString();
+        const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
-        await knex.raw(insertQuery.replace('insert', 'INSERT OR IGNORE'));
+        const releaseEntry = await knex.raw(releaseQuery);
 
-        return curatedReleases;
+        if (release.actors && release.actors.length > 0) {
-    }
+            const actors = await knex('actors').whereIn('name', release.actors);
+            const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
+            const { rows: insertedActors } = newActors.length
+                ? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({ name: actorName })))} ON CONFLICT DO NOTHING RETURNING *`)
+                : { rows: [] };
 
-    return [];
+            await knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({
+                release_id: releaseEntry.rows[0].id,
+                actor_id: actor.id,
+            })), '*');
+        }
+
+        if (release.tags && release.tags.length > 0) {
+            await knex('tags_associated').insert(release.tags.map(tag => ({
+                tag_id: tag,
+                release_id: releaseEntry.rows[0].id,
+            })));
+        }
+    }, []);
 }
 
 async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) {
@@ -148,7 +166,21 @@ async function fetchReleases() {
                 console.log(`${site.name}: Found ${newReleases.length} recent releases, ${upcomingReleases.length} upcoming releases`);
 
                 if (argv.save) {
-                    await storeReleases(newReleases);
+                    const finalReleases = argv.deep
+                        ? await Promise.all(newReleases.map(async (release) => {
+                            if (release.url) {
+                                return fetchScene(release.url);
+                            }
+
+                            return release;
+                        }), {
+                            concurrency: 2,
+                        })
+                        : newReleases;
+
+                    console.log(finalReleases);
+
+                    await storeReleases(finalReleases);
                 }
 
                 return [
@@ -176,8 +208,6 @@ async function fetchReleases() {
         return [];
     }));
 
-    knex.destroy();
-
     const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []);
     const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA));
 

src/fetch-scene.js

@@ -4,6 +4,7 @@ const config = require('config');
 const moment = require('moment');
 
 const knex = require('./knex');
+const argv = require('./argv');
 const scrapers = require('./scrapers');
 
 async function findSite(url) {
@@ -72,6 +73,33 @@ function deriveFilename(scene) {
     return filename;
 }
 
+async function storeRelease(release) {
+    const curatedRelease = {
+        site_id: release.site.id,
+        shoot_id: release.shootId || null,
+        entry_id: release.entryId || null,
+        url: release.url,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        director: release.director,
+        duration: release.duration,
+        likes: release.rating && release.rating.likes,
+        dislikes: release.rating && release.rating.dislikes,
+        rating: release.rating && release.rating.stars,
+    };
+
+    console.log('Saving releases to database');
+    await knex.raw(`${knex('releases').insert(curatedRelease).toString()} ON CONFLICT (site_id, shoot_id) DO UPDATE SET
+        description = EXCLUDED.description,
+        likes = EXCLUDED.likes,
+        dislikes = EXCLUDED.dislikes,
+        rating = EXCLUDED.rating
+    `);
+
+    return release;
+}
+
 async function fetchScene(url) {
     const site = await findSite(url);
     const scraper = scrapers[site.id] || scrapers[site.network.id];
@@ -87,7 +115,11 @@ async function fetchScene(url) {
     const scene = await scraper.fetchScene(url, site);
     const filename = deriveFilename(scene);
 
-    knex.destroy();
+    if (argv.save) {
+        await storeRelease(scene);
+    }
+
+    // knex.destroy();
 
     return {
         ...scene,

src/knex.js

@@ -1,11 +1,19 @@
 'use strict';
 
+const config = require('config');
 const knex = require('knex');
 
+/*
 module.exports = knex({
     client: 'sqlite3',
     connection: {
-        filename: './db.sqlite',
+        filename: path.join(__dirname, '../db.sqlite'),
     },
     useNullAsDefault: true,
 });
+*/
+
+module.exports = knex({
+    client: 'pg',
+    connection: config.database,
+});

src/releases.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const knex = require('./knex');
+
+async function curateRelease(release) {
+    const actors = await knex('actors_associated')
+        .select('actors.id', 'actors.name', 'actors.gender')
+        .where({ release_id: release.id })
+        .leftJoin('actors', 'actors.id', 'actors_associated.actor_id');
+
+    return {
+        id: release.id,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        url: release.url,
+        shootId: release.shoot_id,
+        entryId: release.entry_id,
+        actors,
+        director: release.director,
+        rating: {
+            likes: release.likes,
+            dislikes: release.dislikes,
+            stars: release.stars,
+        },
+        site: {
+            id: release.site_id,
+            name: release.site_name,
+            network: release.network_id,
+        },
+    };
+}
+
+function curateReleases(releases) {
+    return Promise.all(releases.map(async release => curateRelease(release)));
+}
+
+async function fetchReleases() {
+    const releases = await knex('releases')
+        .select('releases.*', 'sites.name as site_name')
+        .leftJoin('sites', 'releases.site_id', 'sites.id')
+        .orderBy('date', 'desc')
+        .limit(100);
+
+    // console.log(curateReleases(releases));
+
+    return curateReleases(releases);
+}
+
+module.exports = {
+    fetchReleases,
+};

src/scrapers/kink.js

@@ -58,7 +58,7 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
         .toDate();
 
     const actors = $(actorsRaw).find('span.names a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
-    const description = $('.shoot-info .description').text();
+    const description = $('.shoot-info .description').text().trim();
 
     const { average: stars } = ratingRes.body;
 

src/tags.js

@@ -3,16 +3,9 @@
 const knex = require('./knex');
 
 async function matchTags(rawTags) {
-    const tagQuery = knex('tags')
+    const tagEntries = await knex('tags').whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase()));
-        .select(knex.raw('ifnull(original.tag, tags.tag) as tag'), knex.raw('ifnull(original.tag, tags.tag) as tag'))
-        .whereIn('tags.tag', rawTags)
-        .leftJoin('tags as original', 'tags.alias_for', 'original.tag')
-        .toString()
-        .replace('where `tags`.`tag` in', 'where `tags`.`tag` collate NOCASE in');
 
-    const tagEntries = await knex.raw(tagQuery);
+    return Array.from(new Set(tagEntries.map((tag => tag.alias_for || tag.tag)).sort())); // reduce to tag name and filter duplicates
-
-    return Array.from(new Set(tagEntries.map(({ tag }) => tag))).sort(); // reduce to tag name and filter duplicates
 }
 
 module.exports = { matchTags };

src/utils/rename.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const path = require('path');
+const Promise = require('bluebird');
+const fs = require('fs-extra');
+const fetchScene = require('../fetch-scene');
+
+const argv = require('../argv');
+
+async function renameFiles() {
+    const filenames = await fs.readdir(process.cwd());
+
+    const curated = await Promise.map(filenames, async (filename) => {
+        const shootId = filename.split(' ')[1];
+        const scene = await fetchScene(`https://kink.com/shoot/${shootId}`);
+
+        if (argv.confirm) {
+            await fs.rename(path.join(process.cwd(), filename), path.join(process.cwd(), `${scene.filename}.mp4`));
+        }
+
+        return scene.filename;
+    }, {
+        concurrency: 5,
+    });
+
+    console.log(curated);
+}
+
+renameFiles();

src/web/releases.js

@@ -0,0 +1,13 @@
+'use strict';
+
+const { fetchReleases } = require('../releases');
+
+async function fetchReleasesApi(req, res) {
+    const releases = await fetchReleases();
+
+    res.render('home', { releases });
+}
+
+module.exports = {
+    fetchReleases: fetchReleasesApi,
+};

src/web/server.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const path = require('path');
+const config = require('config');
+const express = require('express');
+const Router = require('express-promise-router');
+const { createEngine } = require('express-react-views');
+
+const { fetchReleases } = require('./releases');
+
+function initServer() {
+    const app = express();
+    const router = Router();
+
+    app.set('views', path.join(__dirname, '../../assets/views'));
+    app.set('view engine', 'jsx');
+    app.engine('jsx', createEngine());
+
+    router.get('/', fetchReleases);
+
+    app.use(router);
+
+    app.listen(config.web.port, () => {
+        console.log(`Web server listening on port ${config.web.port}`);
+    });
+}
+
+module.exports = initServer;