Major refactor, cleand up site scrape module, fixed and cleaned up release scrape module. Removed old CLI code

src/releases.js

@@ -1,7 +1,17 @@
 'use strict';
 
+const Promise = require('bluebird');
 const knex = require('./knex');
+const argv = require('./argv');
 const whereOr = require('./utils/where-or');
+const { storeTags } = require('./tags');
+const { storeActors } = require('./actors');
+const {
+    createMediaDirectory,
+    storePoster,
+    storePhotos,
+    storeTrailer,
+} = require('./media');
 
 async function curateRelease(release) {
     const [actors, tags, media] = await Promise.all([
@@ -71,6 +81,69 @@ function curateReleases(releases) {
     return Promise.all(releases.map(async release => curateRelease(release)));
 }
 
+function curateScrapedRelease(release) {
+    return {
+        site_id: release.site.id,
+        studio_id: release.studio ? release.studio.id : null,
+        shoot_id: release.shootId || null,
+        entry_id: release.entryId || null,
+        url: release.url,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        // director: release.director,
+        duration: release.duration,
+        likes: release.rating && release.rating.likes,
+        dislikes: release.rating && release.rating.dislikes,
+        rating: release.rating && release.rating.stars && Math.floor(release.rating.stars),
+        deep: Boolean(argv.deep && release.url && !release.upcoming),
+    };
+}
+
+async function storeRelease(release) {
+    const curatedRelease = curateScrapedRelease(release);
+
+    const releaseEntries = await knex('releases')
+        .insert(curatedRelease)
+        .returning('*');
+
+    if (releaseEntries.length) {
+        const releaseEntry = releaseEntries[0];
+
+        console.log(`Stored (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
+
+        await createMediaDirectory(release, releaseEntry.id);
+
+        await Promise.all([
+            storeActors(release, releaseEntry),
+            storeTags(release, releaseEntry),
+            storePhotos(release, releaseEntry),
+            storePoster(release, releaseEntry),
+            storeTrailer(release, releaseEntry),
+        ]);
+
+        return releaseEntry.id;
+    }
+
+    console.error(`Unable to save scene to database, possible collision: "${release.title}" (${release.site.name})`);
+
+    return null;
+}
+
+async function storeReleases(releases) {
+    return Promise.map(releases, async (release) => {
+        try {
+            return storeRelease(release);
+        } catch (error) {
+            console.error(error);
+
+            return null;
+        }
+    }, {
+        concurrency: 2,
+    });
+}
+
 function commonQuery(queryBuilder, {
     filter = [],
     after = new Date(0), // January 1970
@@ -160,4 +233,6 @@ module.exports = {
     fetchSiteReleases,
     fetchNetworkReleases,
     fetchTagReleases,
+    storeRelease,
+    storeReleases,
 };