Replaced network and tag files with SQLite database.

src/scrapers/julesjordan.js

@@ -4,26 +4,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const tagMap = {
-    Anal: 'anal',
-    Asian: 'asian',
-    'Ass To Mouth': 'ATM',
-    'Big Cocks': 'big cock',
-    Black: 'BBC',
-    Blondes: 'blonde',
-    Brunettes: 'brunette',
-    Blowjobs: 'blowjob',
-    Creampie: 'creampie',
-    'Deep Throat': 'deepthroat',
-    Facial: 'facial',
-    Interracial: 'interracial',
-    Lingerie: 'lingerie',
-    Natural: 'natural',
-    'Red Head': 'readhead',
-    'School Girl': 'schoolgirl',
-    Tattoo: 'tattoo',
-    Teen: 'teen',
-};
+const { matchTags } = require('../tags');
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -87,7 +68,7 @@ function scrapeUpcoming(html, site) {
     });
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
     const title = $('.title_bar_hilite').text();
@@ -104,7 +85,7 @@ function scrapeScene(html, url, site) {
     const stars = Number($('.avg_rating').text().trim().replace(/[\s|Avg Rating:]/g, ''));
 
     const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,
@@ -120,7 +101,7 @@ function scrapeScene(html, url, site) {
     };
 }
 
-async function fetchLatest(site) {
+async function fetchLatest(site, _date) {
     const res = await bhttp.get(`${site.url}/trial/categories/movies_1_d.html`);
 
     return scrapeLatest(res.body.toString(), site);

src/scrapers/kink.js

@@ -3,31 +3,9 @@
 const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
-const { kink } = require('../networks');
 
-const tagMap = {
-    Airtight: 'airtight',
-    Anal: 'anal',
-    Asian: 'asian',
-    BDSM: 'BDSM',
-    Blowbang: 'blowbang',
-    Blowjob: 'blowjob',
-    Bondage: 'bondage',
-    Choking: 'choking',
-    'Corporal Punishment': 'corporal punishment',
-    'Double Penetration': 'DP',
-    Gangbang: 'gangbang',
-    'High Heels': 'high heels',
-    Petite: 'petite',
-    'Role Play': 'roleplay',
-    'Rope Bondage': 'bondage',
-    'Rough Sex': 'rough',
-    Shaved: 'shaved',
-    Slapping: 'slapping',
-    'Small Tits': 'small boobs',
-    Squirting: 'squirting',
-    White: 'white',
-};
+const knex = require('../knex');
+const { matchTags } = require('../tags');
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -71,7 +49,6 @@ async function scrapeScene(html, url, id, ratingRes, site) {
     const actorsRaw = $('.shoot-info p.starring');
 
     const sitename = $('.shoot-logo a').attr('href').split('/')[2];
-    const channelSite = kink.sites[sitename];
 
     const date = moment.utc($(actorsRaw)
         .prev()
@@ -87,7 +64,9 @@ async function scrapeScene(html, url, id, ratingRes, site) {
     const { average: stars } = ratingRes.body;
 
     const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+
+    const channelSite = await knex('sites').where({ id: sitename }).first();
+    const tags = await matchTags(rawTags);
 
     return {
         url,

src/scrapers/legalporno.js

@@ -4,6 +4,8 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
+const { matchTags } = require('../tags');
+
 const tagMap = {
     '3+ on 1': 'gangbang',
     anal: 'anal',
@@ -52,7 +54,7 @@ function scrapeLatest(html, site) {
     });
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
 
     const originalTitle = $('h1.watchpage-title').text().trim();
@@ -68,7 +70,7 @@ function scrapeScene(html, url, site) {
     const duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
 
     const rawTags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,

src/scrapers/xempire.js

@@ -4,35 +4,7 @@ const bhttp = require('bhttp');
 const cheerio = require('cheerio');
 const moment = require('moment');
 
-const tagMap = {
-    Anal: 'anal',
-    'Ass Licking': 'ass licking',
-    'Ass To Mouth': 'ATM',
-    'Big Ass': 'big butt',
-    'Big Tits': 'big boobs',
-    Black: 'big black cock',
-    Blonde: 'blonde',
-    Blowjob: 'blowjob',
-    'Blowjob (double)': 'double blowjob',
-    Brunette: 'brunette',
-    'Cum Swallowing': 'swallowing',
-    Cumshot: 'cumshot',
-    Deepthroat: 'deepthroat',
-    'Double Penetration (DP)': 'DP',
-    Ebony: 'ebony',
-    Facial: 'facial',
-    Gangbang: 'gangbang',
-    Gonzo: 'gonzo',
-    Hardcore: 'hardcore',
-    Interracial: 'interracial',
-    Latina: 'latina',
-    Petite: 'petite',
-    'Pussy Licking': 'pussy licking',
-    Rimjob: 'ass licking',
-    'Rough Sex': 'rough',
-    'Small Tits': 'small boobs',
-    Threesome: 'threesome',
-};
+const { matchTags } = require('../tags');
 
 function scrape(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -69,7 +41,7 @@ function scrape(html, site) {
     });
 }
 
-function scrapeSceneFallback($, url, site) {
+async function scrapeSceneFallback($, url, site) {
     const title = $('h1.title').text();
     const date = moment.utc($('.updatedDate').text(), 'MM-DD-YYYY').toDate();
     const actors = $('.sceneColActors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
@@ -78,7 +50,7 @@ function scrapeSceneFallback($, url, site) {
     const stars = $('.currentRating').text().split('/')[0] / 2;
 
     const rawTags = $('.sceneColCategories > a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,
@@ -94,7 +66,7 @@ function scrapeSceneFallback($, url, site) {
     };
 }
 
-function scrapeScene(html, url, site) {
+async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
     const json = $('script[type="application/ld+json"]').html();
 
@@ -122,7 +94,7 @@ function scrapeScene(html, url, site) {
     const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
 
     const rawTags = data.keywords.split(', ');
-    const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
+    const tags = await matchTags(rawTags);
 
     return {
         url,