Added Bang Bros data (no scraper yet). Added 21Sextury scene fetch date fallback.

src/utils/try-links.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+const fs = Promise.promisifyAll(require('fs'));
+
+const knex = require('../knex');
+const argv = require('../argv');
+
+const options = {
+    responseTimeout: 30000,
+};
+
+async function tryLinks() {
+    const sites = await knex('sites').whereIn('network_id', argv.network);
+
+    const results = await Promise.all(sites.map(async (site) => {
+        console.log(`Trying ${site.name} URLs`);
+
+        const [resHttp, resHttpWww, resHttps, resHttpsWww] = await Promise.all([
+            bhttp.get(`http://${site.id}.com/`, options).catch(error => ({ statusCode: error.message })),
+            bhttp.get(`http://www.${site.id}.com/`, options).catch(error => ({ statusCode: error.message })),
+            bhttp.get(`https://${site.id}.com/`, options).catch(error => ({ statusCode: error.message })),
+            bhttp.get(`https://www.${site.id}.com/`, options).catch(error => ({ statusCode: error.message })),
+        ]);
+
+        console.log(`Got results for ${site.name}`);
+
+        return {
+            ...site,
+            url: (resHttp.statusCode === 200 && `http://${site.id}.com`)
+            || (resHttpWww.statusCode === 200 && `http://www.${site.id}.com`)
+            || (resHttps.statusCode === 200 && `https://${site.id}.com`)
+            || (resHttpsWww.statusCode === 200 && `https://www.${site.id}.com`)
+            || site.url,
+            network_id: site.network_id,
+        };
+    }));
+
+    const sortedResults = results.sort((siteA, siteB) => {
+        if (siteA.id > siteB.id) return 1;
+        if (siteA.id < siteB.id) return -1;
+
+        return 0;
+    });
+
+    console.log(sortedResults);
+    await fs.writeFileAsync('./src/utils/link-results.json', JSON.stringify(sortedResults, null, 4));
+}
+
+tryLinks();