Saving results to database. Showing webpage.

src/app.js

@@ -5,6 +5,7 @@ const clipboard = require('clipboardy');
 
 const argv = require('./argv');
 const { renderReleases, renderScene } = require('./tui/render');
+const initServer = require('./web/server');
 
 const fetchReleases = require('./fetch-releases');
 const fetchScene = require('./fetch-scene');
@@ -34,35 +35,43 @@ function getMethod() {
         };
     }
 
-    return {
-        fetch: () => fetchReleases(),
-        render: renderReleases,
-    };
+    if (argv.fetch) {
+        return {
+            fetch: () => fetchReleases(),
+            render: renderReleases,
+        };
+    }
+
+    return null;
 }
 
 async function init() {
-    const screen = argv.render && !argv.filename && initScreen();
+    initServer();
 
+    const screen = argv.render && !argv.filename && initScreen();
 
     try {
         const method = getMethod();
-        const result = await method.fetch();
 
-        if (result) {
-            if (argv.copy && result.copy) {
-                clipboard.writeSync(result.copy);
-                console.log(`Result copied to clipboard: ${result.copy}`);
-            }
+        if (method) {
+            const result = await method.fetch();
 
-            if (argv.filename && result.filename) {
-                console.log(result.filename);
+            if (result) {
+                if (argv.copy && result.copy) {
+                    clipboard.writeSync(result.copy);
+                    console.log(`Result copied to clipboard: ${result.copy}`);
+                }
 
-                // setTimeout(() => log(), 5000);
-                return;
-            }
+                if (argv.filename && result.filename) {
+                    console.log(result.filename);
 
-            if (argv.render) {
-                method.render(result, screen);
+                    // setTimeout(() => log(), 5000);
+                    return;
+                }
+
+                if (argv.render) {
+                    method.render(result, screen);
+                }
             }
         }
     } catch (error) {

src/argv.js

@@ -5,6 +5,16 @@ const yargs = require('yargs');
 
 const { argv } = yargs
     .command('npm start')
+    .option('fetch', {
+        describe: 'Fetch latest releases',
+        type: 'boolean',
+        default: false,
+    })
+    .option('deep', {
+        describe: 'Fetch details for all releases',
+        type: 'boolean',
+        default: false,
+    })
     .option('networks', {
         describe: 'Networks to include (overrides config)',
         type: 'array',
@@ -23,12 +33,12 @@ const { argv } = yargs
     .option('save', {
         describe: 'Save fetched releases to database',
         type: 'boolean',
-        default: true,
+        default: false,
     })
     .option('render', {
         describe: 'Fetch data without rendering interface',
         type: 'boolean',
-        default: true,
+        default: false,
     })
     .option('scene', {
         describe: 'Fetch scene info from URL',

src/fetch-releases.js

@@ -1,11 +1,13 @@
 'use strict';
 
 const config = require('config');
+const Promise = require('bluebird');
 const moment = require('moment');
 
 const argv = require('./argv');
 const knex = require('./knex');
 const scrapers = require('./scrapers');
+const fetchScene = require('./fetch-scene');
 
 function destructConfigNetworks(networks) {
     return networks.reduce((acc, network) => {
@@ -72,32 +74,48 @@ async function findDuplicateReleases(latestReleases, _siteId) {
         .orWhereIn('entry_id', latestReleasesEntryIds);
 }
 
-async function storeReleases(releases) {
-    const curatedReleases = releases.map(release => ({
-        site_id: release.site.id,
-        shoot_id: release.shootId || null,
-        entry_id: release.entryId || null,
-        url: release.url,
-        title: release.title,
-        date: release.date,
-        description: release.description,
-        director: release.director,
-        duration: release.duration,
-        likes: release.rating && release.rating.likes,
-        dislikes: release.rating && release.rating.dislikes,
-        rating: release.rating && release.rating.stars,
-    }));
+async function storeReleases(releases = []) {
+    return Promise.reduce(releases, async (acc, release) => {
+        await acc;
 
-    if (curatedReleases.length) {
-        console.log(`Saving ${curatedReleases.length} new releases to database`);
+        const curatedRelease = {
+            site_id: release.site.id,
+            shoot_id: release.shootId || null,
+            entry_id: release.entryId || null,
+            url: release.url,
+            title: release.title,
+            date: release.date,
+            description: release.description,
+            director: release.director,
+            duration: release.duration,
+            likes: release.rating && release.rating.likes,
+            dislikes: release.rating && release.rating.dislikes,
+            rating: release.rating && release.rating.stars,
+        };
 
-        const insertQuery = knex('releases').insert(curatedReleases).toString();
-        await knex.raw(insertQuery.replace('insert', 'INSERT OR IGNORE'));
+        const releaseQuery = `${knex('releases').insert(curatedRelease).toString()} ON CONFLICT DO NOTHING RETURNING *`;
+        const releaseEntry = await knex.raw(releaseQuery);
 
-        return curatedReleases;
-    }
+        if (release.actors && release.actors.length > 0) {
+            const actors = await knex('actors').whereIn('name', release.actors);
+            const newActors = release.actors.filter(actorName => !actors.some(actor => actor.name === actorName));
+            const { rows: insertedActors } = newActors.length
+                ? await knex.raw(`${knex('actors').insert(newActors.map(actorName => ({ name: actorName })))} ON CONFLICT DO NOTHING RETURNING *`)
+                : { rows: [] };
 
-    return [];
+            await knex('actors_associated').insert(actors.concat(insertedActors).map(actor => ({
+                release_id: releaseEntry.rows[0].id,
+                actor_id: actor.id,
+            })), '*');
+        }
+
+        if (release.tags && release.tags.length > 0) {
+            await knex('tags_associated').insert(release.tags.map(tag => ({
+                tag_id: tag,
+                release_id: releaseEntry.rows[0].id,
+            })));
+        }
+    }, []);
 }
 
 async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page = 1) {
@@ -148,7 +166,21 @@ async function fetchReleases() {
                 console.log(`${site.name}: Found ${newReleases.length} recent releases, ${upcomingReleases.length} upcoming releases`);
 
                 if (argv.save) {
-                    await storeReleases(newReleases);
+                    const finalReleases = argv.deep
+                        ? await Promise.all(newReleases.map(async (release) => {
+                            if (release.url) {
+                                return fetchScene(release.url);
+                            }
+
+                            return release;
+                        }), {
+                            concurrency: 2,
+                        })
+                        : newReleases;
+
+                    console.log(finalReleases);
+
+                    await storeReleases(finalReleases);
                 }
 
                 return [
@@ -176,8 +208,6 @@ async function fetchReleases() {
         return [];
     }));
 
-    knex.destroy();
-
     const accumulatedScenes = scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []);
     const sortedScenes = accumulatedScenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA));
 

src/fetch-scene.js

@@ -4,6 +4,7 @@ const config = require('config');
 const moment = require('moment');
 
 const knex = require('./knex');
+const argv = require('./argv');
 const scrapers = require('./scrapers');
 
 async function findSite(url) {
@@ -72,6 +73,33 @@ function deriveFilename(scene) {
     return filename;
 }
 
+async function storeRelease(release) {
+    const curatedRelease = {
+        site_id: release.site.id,
+        shoot_id: release.shootId || null,
+        entry_id: release.entryId || null,
+        url: release.url,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        director: release.director,
+        duration: release.duration,
+        likes: release.rating && release.rating.likes,
+        dislikes: release.rating && release.rating.dislikes,
+        rating: release.rating && release.rating.stars,
+    };
+
+    console.log('Saving releases to database');
+    await knex.raw(`${knex('releases').insert(curatedRelease).toString()} ON CONFLICT (site_id, shoot_id) DO UPDATE SET
+        description = EXCLUDED.description,
+        likes = EXCLUDED.likes,
+        dislikes = EXCLUDED.dislikes,
+        rating = EXCLUDED.rating
+    `);
+
+    return release;
+}
+
 async function fetchScene(url) {
     const site = await findSite(url);
     const scraper = scrapers[site.id] || scrapers[site.network.id];
@@ -87,7 +115,11 @@ async function fetchScene(url) {
     const scene = await scraper.fetchScene(url, site);
     const filename = deriveFilename(scene);
 
-    knex.destroy();
+    if (argv.save) {
+        await storeRelease(scene);
+    }
+
+    // knex.destroy();
 
     return {
         ...scene,

src/knex.js

@@ -1,11 +1,19 @@
 'use strict';
 
+const config = require('config');
 const knex = require('knex');
 
+/*
 module.exports = knex({
     client: 'sqlite3',
     connection: {
-        filename: './db.sqlite',
+        filename: path.join(__dirname, '../db.sqlite'),
     },
     useNullAsDefault: true,
 });
+*/
+
+module.exports = knex({
+    client: 'pg',
+    connection: config.database,
+});

src/releases.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const knex = require('./knex');
+
+async function curateRelease(release) {
+    const actors = await knex('actors_associated')
+        .select('actors.id', 'actors.name', 'actors.gender')
+        .where({ release_id: release.id })
+        .leftJoin('actors', 'actors.id', 'actors_associated.actor_id');
+
+    return {
+        id: release.id,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        url: release.url,
+        shootId: release.shoot_id,
+        entryId: release.entry_id,
+        actors,
+        director: release.director,
+        rating: {
+            likes: release.likes,
+            dislikes: release.dislikes,
+            stars: release.stars,
+        },
+        site: {
+            id: release.site_id,
+            name: release.site_name,
+            network: release.network_id,
+        },
+    };
+}
+
+function curateReleases(releases) {
+    return Promise.all(releases.map(async release => curateRelease(release)));
+}
+
+async function fetchReleases() {
+    const releases = await knex('releases')
+        .select('releases.*', 'sites.name as site_name')
+        .leftJoin('sites', 'releases.site_id', 'sites.id')
+        .orderBy('date', 'desc')
+        .limit(100);
+
+    // console.log(curateReleases(releases));
+
+    return curateReleases(releases);
+}
+
+module.exports = {
+    fetchReleases,
+};

src/scrapers/kink.js

@@ -58,7 +58,7 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
         .toDate();
 
     const actors = $(actorsRaw).find('span.names a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
-    const description = $('.shoot-info .description').text();
+    const description = $('.shoot-info .description').text().trim();
 
     const { average: stars } = ratingRes.body;
 

src/tags.js

@@ -3,16 +3,9 @@
 const knex = require('./knex');
 
 async function matchTags(rawTags) {
-    const tagQuery = knex('tags')
-        .select(knex.raw('ifnull(original.tag, tags.tag) as tag'), knex.raw('ifnull(original.tag, tags.tag) as tag'))
-        .whereIn('tags.tag', rawTags)
-        .leftJoin('tags as original', 'tags.alias_for', 'original.tag')
-        .toString()
-        .replace('where `tags`.`tag` in', 'where `tags`.`tag` collate NOCASE in');
+    const tagEntries = await knex('tags').whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase()));
 
-    const tagEntries = await knex.raw(tagQuery);
-
-    return Array.from(new Set(tagEntries.map(({ tag }) => tag))).sort(); // reduce to tag name and filter duplicates
+    return Array.from(new Set(tagEntries.map((tag => tag.alias_for || tag.tag)).sort())); // reduce to tag name and filter duplicates
 }
 
 module.exports = { matchTags };

src/utils/rename.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const path = require('path');
+const Promise = require('bluebird');
+const fs = require('fs-extra');
+const fetchScene = require('../fetch-scene');
+
+const argv = require('../argv');
+
+async function renameFiles() {
+    const filenames = await fs.readdir(process.cwd());
+
+    const curated = await Promise.map(filenames, async (filename) => {
+        const shootId = filename.split(' ')[1];
+        const scene = await fetchScene(`https://kink.com/shoot/${shootId}`);
+
+        if (argv.confirm) {
+            await fs.rename(path.join(process.cwd(), filename), path.join(process.cwd(), `${scene.filename}.mp4`));
+        }
+
+        return scene.filename;
+    }, {
+        concurrency: 5,
+    });
+
+    console.log(curated);
+}
+
+renameFiles();

src/web/releases.js

@@ -0,0 +1,13 @@
+'use strict';
+
+const { fetchReleases } = require('../releases');
+
+async function fetchReleasesApi(req, res) {
+    const releases = await fetchReleases();
+
+    res.render('home', { releases });
+}
+
+module.exports = {
+    fetchReleases: fetchReleasesApi,
+};

src/web/server.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const path = require('path');
+const config = require('config');
+const express = require('express');
+const Router = require('express-promise-router');
+const { createEngine } = require('express-react-views');
+
+const { fetchReleases } = require('./releases');
+
+function initServer() {
+    const app = express();
+    const router = Router();
+
+    app.set('views', path.join(__dirname, '../../assets/views'));
+    app.set('view engine', 'jsx');
+    app.engine('jsx', createEngine());
+
+    router.get('/', fetchReleases);
+
+    app.use(router);
+
+    app.listen(config.web.port, () => {
+        console.log(`Web server listening on port ${config.web.port}`);
+    });
+}
+
+module.exports = initServer;