Added Brazzers scraper. Removed scene-only sites in database in favor of automatic network fallback.

src/fetch-scene.js

@@ -9,14 +9,16 @@ const scrapers = require('./scrapers');
 async function findSite(url) {
     const { origin } = new URL(url);
 
-    const site = await knex('sites').where({ url: origin }).first();
+    const site = await knex('sites').where({ url: origin }).first()
+        // scenes might be listed on network site, let network scraper find channel site
+        || await knex('networks').where({ url: origin }).first();
 
     return {
         id: site.id,
         name: site.name,
         description: site.description,
         url: site.url,
-        networkId: site.network_id,
+        networkId: site.network_id || site.id,
     };
 }
 

src/scrapers/brazzers.js

@@ -0,0 +1,118 @@
+'use strict';
+
+/* eslint-disable */
+const bhttp = require('bhttp');
+const cheerio = require('cheerio');
+const moment = require('moment');
+
+const knex = require('../knex');
+const { matchTags } = require('../tags');
+
+function scrape(html, site, upcoming) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+    const sceneElements = $('.release-card.scene').toArray();
+
+    return sceneElements.reduce((acc, element) => {
+        const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
+
+        if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
+            return acc;
+        }
+
+        const sceneLinkElement = $(element).find('a');
+
+        const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
+        const title = sceneLinkElement.attr('title');
+        const shootId = url.split('/').slice(-3, -2)[0];
+
+        const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
+        const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
+
+        const likes = Number($(element).find('.label-rating .like-amount').text());
+        const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
+
+        return acc.concat({
+            url,
+            shootId,
+            title,
+            actors,
+            date,
+            rating: {
+                likes,
+                dislikes,
+            },
+            site,
+        });
+    }, []);
+}
+
+async function scrapeScene(html, url, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+
+    const shootId = url.split('/').slice(-3, -2)[0];
+    const title = $('.scene-title[itemprop="name"]').text();
+
+    const description = $('#scene-description p[itemprop="description"]')
+        .contents()
+        .first()
+        .text()
+        .trim();
+
+    const date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
+    const actors = $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
+    const duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
+
+    const likes = Number($('.label-rating .like').text());
+    const dislikes = Number($('.label-rating .dislike').text());
+
+    const siteElement = $('.niche-site-logo');
+    const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
+    const siteName = siteElement.attr('title');
+
+    const rawTags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
+
+    const [tags, channelSite] = await Promise.all([
+        matchTags(rawTags),
+        knex('sites')
+            .where({ url: siteUrl })
+            .orWhere({ name: siteName })
+            .first(),
+    ]);
+
+    return {
+        url,
+        shootId,
+        title,
+        actors,
+        date,
+        rating: {
+            likes,
+            dislikes,
+        },
+        site: channelSite || site,
+    };
+}
+
+async function fetchLatest(site) {
+    const res = await bhttp.get(`${site.url}/`);
+
+    return scrape(res.body.toString(), site, false);
+}
+
+async function fetchUpcoming(site) {
+    const res = await bhttp.get(`${site.url}/`);
+
+    return scrape(res.body.toString(), site, true);
+}
+
+async function fetchScene(url, site) {
+    const res = await bhttp.get(url);
+
+    return scrapeScene(res.body.toString(), url, site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchUpcoming,
+    fetchScene,
+};

src/scrapers/index.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const brazzers = require('./brazzers');
 const julesjordan = require('./julesjordan');
 const kink = require('./kink');
 const legalporno = require('./legalporno');
@@ -9,6 +10,7 @@ const vixen = require('./vixen');
 const xempire = require('./xempire');
 
 module.exports = {
+    brazzers,
     julesjordan,
     kink,
     legalporno,

src/scrapers/private.js

@@ -57,13 +57,16 @@ async function scrapeScene(html, url, site) {
     const siteElement = $('.content-wrapper .logos-sites a');
     const siteUrl = siteElement.attr('href').slice(0, -1);
     const siteName = siteElement.text();
-    const channelSite = await knex('sites')
-        .where({ url: siteUrl })
-        .orWhere({ name: siteName })
-        .first();
 
     const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
-    const tags = await matchTags(rawTags);
+
+    const [tags, channelSite] = await Promise.all([
+        matchTags(rawTags),
+        knex('sites')
+            .where({ url: siteUrl })
+            .orWhere({ name: siteName })
+            .first(),
+    ]);
 
     return {
         url,

src/scrapers/template.js

@@ -9,7 +9,7 @@ const { matchTags } = require('../tags');
 
 function scrapeLatest(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElements = $('.scenes-latest');
+    const sceneElements = $('.scenes-latest').toArray();
 
     return sceneElements.map((element) => {
         return {
@@ -30,7 +30,7 @@ function scrapeLatest(html, site) {
 
 function scrapeUpcoming(html, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const sceneElements = $('.scenes-upcoming');
+    const sceneElements = $('.scenes-upcoming').toArray();
 
     return sceneElements.map((element) => {
         return {