Wrapping up

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 dist/
+log/
 public/js/*
 public/css/*
 config/*

config/default.js

@@ -52,37 +52,6 @@ module.exports = {
         'xempire',
     ],
     fetchAfter: [1, 'week'],
-    columns: [
-        {
-            value: 'date',
-            format: 'MMM DD, YYYY',
-            width: 14,
-        },
-        {
-            value: 'network',
-            width: 15,
-        },
-        {
-            value: 'site',
-            width: 25,
-        },
-        {
-            value: 'shootId',
-            width: 15,
-        },
-        {
-            value: 'title',
-            width: 80,
-        },
-        {
-            value: 'actors',
-            width: 60,
-        },
-        {
-            value: 'rating',
-            width: 30,
-        },
-    ],
     media: {
         path: './',
         thumbnailSize: 320, // width for 16:9 will be exactly 576px

log/2020-01-13.log

@@ -1891,3 +1891,25 @@
 2020-01-13T22:14:58.307Z debug: 'actors_avatars: Updating 0'
 2020-01-13T22:14:58.309Z debug: 'actors_photos: Inserting 1'
 2020-01-13T22:14:58.309Z debug: 'actors_photos: Updating 0'
+2020-01-13T22:54:13.451Z debug: 'networks: Inserting 1'
+2020-01-13T22:54:13.454Z debug: 'networks: Updating 23'
+2020-01-13T22:54:13.569Z debug: 'sites: Inserting 5'
+2020-01-13T22:54:13.570Z debug: 'sites: Updating 389'
+2020-01-13T22:54:14.254Z debug: 'studios: Inserting 0'
+2020-01-13T22:54:14.255Z debug: 'studios: Updating 22'
+2020-01-13T22:54:14.316Z debug: 'tags_groups: Inserting 0'
+2020-01-13T22:54:14.317Z debug: 'tags_groups: Updating 13'
+2020-01-13T22:54:14.367Z debug: 'tags: Inserting 0'
+2020-01-13T22:54:14.368Z debug: 'tags: Updating 143'
+2020-01-13T22:54:14.613Z debug: 'tags: Inserting 0'
+2020-01-13T22:54:14.614Z debug: 'tags: Updating 165'
+2020-01-13T22:54:14.863Z debug: 'sites_tags: Inserting 0'
+2020-01-13T22:54:14.864Z debug: 'sites_tags: Updating 13'
+2020-01-13T22:54:14.896Z debug: 'media: Inserting 0'
+2020-01-13T22:54:14.897Z debug: 'media: Updating 48'
+2020-01-13T22:54:14.972Z debug: 'tags_posters: Inserting 0'
+2020-01-13T22:54:14.972Z debug: 'tags_posters: Updating 28'
+2020-01-13T22:54:14.979Z debug: 'tags_photos: Inserting 0'
+2020-01-13T22:54:14.979Z debug: 'tags_photos: Updating 20'
+2020-01-13T22:54:15.053Z debug: 'countries: Inserting 0'
+2020-01-13T22:54:15.053Z debug: 'countries: Updating 250'

seeds/00_networks.js

@@ -8,6 +8,11 @@ const networks = [
         url: 'https://www.21sextury.com',
         description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.',
     },
+    {
+        slug: 'babes',
+        name: 'Babes',
+        url: 'https://www.babes.com',
+    },
     {
         slug: 'bang',
         name: 'Bang!',

seeds/01_sites.js

@@ -119,6 +119,43 @@ function getSites(networksMap) {
             name: 'Sweet Sophie Moone',
             network_id: networksMap['21sextury'],
         },
+        // BABES
+        {
+            name: 'Babes',
+            url: 'https://www.babes.com/scenes?site=213',
+            slug: 'babes',
+            network_id: networksMap.babes,
+        },
+        {
+            name: 'Babes Unleashed',
+            url: 'https://www.babes.com/scenes?site=218',
+            slug: 'babesunleashed',
+            network_id: networksMap.babes,
+        },
+        {
+            name: 'Black Is Better',
+            url: 'https://www.babes.com/scenes?site=217',
+            slug: 'blackisbetter',
+            network_id: networksMap.babes,
+        },
+        {
+            name: 'Elegant Anal',
+            url: 'https://www.babes.com/scenes?site=216',
+            slug: 'elegantanal',
+            network_id: networksMap.babes,
+        },
+        {
+            name: 'Office Obsession',
+            url: 'https://www.babes.com/scenes?site=214',
+            slug: 'officeobsession',
+            network_id: networksMap.babes,
+        },
+        {
+            name: 'Step Mom Lessons',
+            url: 'https://www.babes.com/scenes?site=215',
+            slug: 'stepmomlessons',
+            network_id: networksMap.babes,
+        },
         // BANG
         {
             name: 'Trickery',

src/scrapers/babes.js

@@ -0,0 +1,154 @@
+'use strict';
+
+/* eslint-disable newline-per-chained-call */
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
+const moment = require('moment');
+
+const { cookieToData } = require('../utils/cookies');
+
+function getThumbs(scene) {
+    if (scene.images.poster) {
+        return scene.images.poster.map(image => image.xl.url);
+    }
+
+    if (scene.images.card_main_rect) {
+        return scene.images.card_main_rect
+            .concat(scene.images.card_secondary_rect || [])
+            .map(image => image.xl.url.replace('.thumb', ''));
+    }
+
+    return [];
+}
+
+async function scrapeLatest(items, site) {
+    return Promise.all(items.map(async (data) => {
+        const { id: entryId, title, description } = data;
+        const url = `https://www.babes.com/scene/${entryId}/`;
+        const date = new Date(data.dateReleased);
+        const actors = data.actors.map(actor => actor.name);
+
+        const tags = data.tags.map(tag => tag.name);
+
+        const [poster, ...photos] = getThumbs(data);
+        const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
+        const duration = data.videos.mediabook && data.videos.mediabook.length;
+
+        return {
+            url,
+            entryId,
+            title,
+            description,
+            actors,
+            tags,
+            duration,
+            poster,
+            photos,
+            trailer: trailer && {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
+            },
+            date,
+            site,
+        };
+    }));
+}
+
+async function scrapeScene(data, url, site) {
+    const { id: entryId, title, description } = data;
+    const date = new Date(data.dateReleased);
+    const actors = data.actors.map(actor => actor.name);
+
+    const tags = data.tags.map(tag => tag.name);
+
+    const [poster, ...photos] = getThumbs(data);
+    const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
+
+    const siteName = data.collections[0].name;
+    const channel = siteName.replace(/\s+/g, '').toLowerCase();
+
+    return {
+        url,
+        entryId,
+        title,
+        description,
+        actors,
+        tags,
+        poster,
+        photos,
+        trailer: trailer && {
+            src: trailer.urls.view,
+            quality: parseInt(trailer.format, 10),
+        },
+        date,
+        site,
+        channel,
+    };
+}
+
+function getUrl(site) {
+    const { hostname, search } = new URL(site.url);
+
+    if (hostname.match(/(www\.)?babes\.com/) && search.match(/\?site=\d+/)) {
+        return site.url;
+    }
+
+    if (site.parameters && site.parameters.siteId) {
+        return `https://www.babes.com/scenes?site=${site.parameters.siteId}`;
+    }
+
+    throw new Error(`Babes site '${site.name}' (${site.url}) not supported`);
+}
+
+async function fetchLatest(site, page = 1) {
+    const url = getUrl(site);
+    const { search } = new URL(url);
+    const siteId = new URLSearchParams(search).get('site');
+
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    console.log(url, siteId);
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
+    const limit = 10;
+    const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
+    const res = await session.get(apiUrl, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeLatest(res.body.result, site);
+}
+
+async function fetchScene(url, site) {
+    const entryId = url.match(/\d+/)[0];
+
+    const cookieJar = new CookieJar();
+    const session = bhttp.session({ cookieJar });
+
+    await session.get(url);
+
+    const cookieString = await cookieJar.getCookieStringAsync(url);
+    const { instance_token: instanceToken } = cookieToData(cookieString);
+
+    const res = await session.get(`https://site-api.project1service.com/v2/releases/${entryId}`, {
+        headers: {
+            Instance: instanceToken,
+        },
+    });
+
+    return scrapeScene(res.body.result, url, site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchScene,
+};

src/scrapers/scrapers.js

@@ -2,6 +2,7 @@
 
 // releases
 const twentyonesextury = require('./21sextury');
+const babes = require('./babes');
 const bang = require('./bang');
 const bangbros = require('./bangbros');
 const blowpass = require('./blowpass');
@@ -34,6 +35,7 @@ const pornhub = require('./pornhub');
 module.exports = {
     releases: {
         '21sextury': twentyonesextury,
+        babes,
         bang,
         bangbros,
         blowpass,

src/sites.js

@@ -63,18 +63,17 @@ async function findSiteByUrl(url) {
     const { hostname } = new URL(url);
     const domain = hostname.replace(/www.|tour./, '');
 
-    const site = await knex('sites')
+    const sites = await knex('sites')
         .leftJoin('networks', 'sites.network_id', 'networks.id')
         .select(
             'sites.*',
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
-        .where('sites.url', 'like', `%${domain}`)
-        .orWhere('sites.url', url)
-        .first();
+        .where('sites.url', 'like', `${domain}`)
+        .orWhere('sites.url', url);
 
-    if (site) {
-        const curatedSite = curateSite(site, true);
+    if (sites.length > 0) {
+        const curatedSite = curateSite(sites[0], true);
 
         return curatedSite;
     }