Added Vixen scraper. Added LegalPorno studio IDs.

src/fetch-releases.js

@@ -47,8 +47,40 @@ async function accumulateIncludedSites() {
     return curateSites(rawSites);
 }
 
+async function getExistingReleases() {
+    return knex('releases');
+    // .where('date', '>', new Date(2019, 2, 26));
+}
+
+async function storeReleases(releases) {
+    const curatedReleases = releases.map(release => ({
+        site_id: release.site.id,
+        shoot_id: release.shootId || null,
+        url: release.url,
+        title: release.title,
+        date: release.date,
+        description: release.description,
+        duration: release.duration,
+        likes: release.rating && release.rating.likes,
+        dislikes: release.rating && release.rating.dislikes,
+        rating: release.rating && release.rating.stars,
+    }));
+
+    if (curatedReleases.length) {
+        console.log(`Adding ${curatedReleases.length} releases to database (if unique)`);
+
+        const insertQuery = knex('releases').insert(curatedReleases).toString();
+        await knex.raw(insertQuery.replace('insert', 'INSERT OR IGNORE'));
+
+        return curatedReleases;
+    }
+
+    return [];
+}
+
 async function fetchReleases() {
     const sites = await accumulateIncludedSites();
+    const releases = await getExistingReleases();
 
     const scenesPerSite = await Promise.all(sites.map(async (site) => {
         const scraper = scrapers[site.id] || scrapers[site.networkId];
@@ -59,6 +91,10 @@ async function fetchReleases() {
                 scraper.fetchUpcoming ? scraper.fetchUpcoming(site) : [],
             ]);
 
+            console.log(`${latest.length} published releases and ${upcoming.length} upcoming releases found`);
+
+            await storeReleases(latest, releases);
+
             return [...latest, ...upcoming];
         }
 

src/scrapers/boilerplate.js

@@ -5,7 +5,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const tagMap = {};
+const { matchTags } = require('../tags');
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });

src/scrapers/index.js

@@ -5,6 +5,7 @@ const julesjordan = require('./julesjordan');
 const kink = require('./kink');
 const legalporno = require('./legalporno');
 const pervcity = require('./pervcity');
+const vixen = require('./vixen');
 
 module.exports = {
     xempire,
@@ -12,4 +13,5 @@ module.exports = {
     kink,
     legalporno,
     pervcity,
+    vixen,
 };

src/scrapers/legalporno.js

@@ -8,7 +8,7 @@ const { matchTags } = require('../tags');
 
 function extractTitle(originalTitle) {
     const titleComponents = originalTitle.split(' ');
-    const sceneIdMatch = titleComponents.slice(-1)[0].match(/(GP|SZ|IV|GIO|AA|GL|BZ|FS)\d+/); // detect studio prefixes
+    const sceneIdMatch = titleComponents.slice(-1)[0].match(/(GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS)\d+/); // detect studio prefixes
     const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
     const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;