Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.

src/utils/cookies.js

@@ -0,0 +1,16 @@
+'use strict';
+
+function cookieToData(cookieString) {
+    return cookieString.split('; ').reduce((acc, cookie) => {
+        const [key, value] = cookie.split('=');
+
+        return {
+            ...acc,
+            [key]: value,
+        };
+    }, {});
+}
+
+module.exports = {
+    cookieToData,
+};

src/utils/mofos.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+
+const knex = require('../knex');
+
+async function run() {
+    const network = await knex('networks').where('slug', 'mofos').first();
+    const sites = await knex('sites').where('network_id', network.id);
+
+    await Promise.map(sites, async (site) => {
+        const res = await bhttp.get(site.url);
+
+        console.log(site.url, res.statusCode);
+    }, {
+        concurrency: 5,
+    });
+}
+
+run();

src/utils/upsert.js

@@ -35,7 +35,7 @@ async function upsert(table, items, duplicatesById, identifier = 'id', knex) {
         return Promise.all([
             knex(table).insert(insert),
             knex.transaction(async trx => Promise.all(update.map(item => trx
-                .where({ id: item.id })
+                .where({ [identifier]: item[identifier] })
                 .update(item)
                 .into(table)))),
         ]);