Added support for Reality Kings sites 'Look At Her Now' and 'Tranny Surprise'.

src/argv.js

@@ -25,7 +25,7 @@ const { argv } = yargs
         alias: 'actor',
     })
     .option('sources', {
-        describe: 'Only use these scrapers for actor data',
+        describe: 'Use these scrapers for actor data',
         type: 'array',
         alias: 'source',
     })

src/scrape-release.js

@@ -30,12 +30,15 @@ async function findSite(url, release) {
 
 async function scrapeRelease(url, release, deep = false) {
     const site = await findSite(url, release);
-    const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
+
+    console.log(url, site);
 
     if (!site) {
         throw new Error('Could not find site in database');
     }
 
+    const scraper = scrapers.releases[site.slug] || scrapers.releases[site.network.slug];
+
     if (!scraper) {
         throw new Error('Could not find scraper for URL');
     }

src/scrapers/realitykings.js

@@ -4,11 +4,10 @@
 const Promise = require('bluebird');
 const bhttp = require('bhttp');
 const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
+const cheerio = require('cheerio');
 const moment = require('moment');
 
-const { fetchSites } = require('../sites');
 const { cookieToData } = require('../utils/cookies');
-const { matchTags } = require('../tags');
 
 function getThumbs(scene) {
     if (scene.images.poster) {
@@ -31,11 +30,11 @@ async function scrapeLatest(items, site) {
         const date = new Date(data.dateReleased);
         const actors = data.actors.map(actor => actor.name);
 
-        const rawTags = data.tags.map(tag => tag.name);
-        const tags = await matchTags(rawTags);
+        const tags = data.tags.map(tag => tag.name);
 
         const [poster, ...photos] = getThumbs(data);
         const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
+        const duration = data.videos.mediabook && data.videos.mediabook.length;
 
         return {
             url,
@@ -44,6 +43,7 @@ async function scrapeLatest(items, site) {
             description,
             actors,
             tags,
+            duration,
             poster,
             photos,
             trailer: trailer && {
@@ -56,31 +56,62 @@ async function scrapeLatest(items, site) {
     }));
 }
 
+function scrapeLatestClassic(html, site) {
+    const $ = cheerio.load(html, { normalizeWhitespace: true });
+
+    const stateTag = $('script:contains("initialState")').html();
+    const prefix = 'initialState = {';
+    const prefixIndex = stateTag.indexOf('initialState = {');
+    const suffix = '};';
+    const stateString = stateTag.slice(prefixIndex + prefix.length - 1, stateTag.indexOf('};', prefixIndex) + suffix.length - 1);
+    const data = JSON.parse(stateString);
+
+    return Object.values(data.entities.releases).map((scene) => {
+        const release = { site };
+
+        release.entryId = scene.id;
+        release.url = `${site.url}/scene/${release.entryId}/`;
+
+        release.title = scene.title;
+        release.description = scene.description;
+        release.date = new Date(scene.dateReleased);
+
+        release.actors = scene.actors.map(actorId => data.entities.actors[actorId].name);
+        release.tags = scene.tags.map(tagId => data.entities.tags[tagId].name);
+
+        const [poster, ...photos] = getThumbs(scene);
+
+        release.poster = poster;
+        release.photos = photos;
+
+        if (scene.videos.mediabook) {
+            const trailer = (scene.videos.mediabook.files['720p'] || scene.videos.mediabook.files['320p']);
+
+            release.duration = scene.videos.mediabook.length;
+            release.trailer = trailer && {
+                src: trailer.urls.view,
+                quality: parseInt(trailer.format, 10),
+            };
+        }
+
+        return release;
+    });
+}
+
 async function scrapeScene(data, url, site) {
     const { id: entryId, title, description } = data;
     const date = new Date(data.dateReleased);
     const actors = data.actors.map(actor => actor.name);
 
-    const rawTags = data.tags.map(tag => tag.name);
+    const tags = data.tags.map(tag => tag.name);
 
     const [poster, ...photos] = getThumbs(data);
     const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
 
     const siteName = data.collections[0].name;
-    const siteId = data.collections[0].id;
-    const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
-    const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
-
-    const [[channelSite], tags] = await Promise.all([
-        site.isFallback
-            ? fetchSites({
-                slug: siteSlug,
-                name: siteName,
-                url: siteUrl,
-            })
-            : [site],
-        matchTags(rawTags),
-    ]);
+    // const siteId = data.collections[0].id;
+    const channel = siteName.replace(/\s+/g, '').toLowerCase();
+    // const siteUrl = `https://www.realitykings.com/scenes?site=${siteId}`;
 
     return {
         url,
@@ -96,7 +127,8 @@ async function scrapeScene(data, url, site) {
             quality: parseInt(trailer.format, 10),
         },
         date,
-        site: channelSite,
+        site,
+        channel,
     };
 }
 
@@ -114,7 +146,21 @@ function getUrl(site) {
     throw new Error(`Reality Kings site '${site.name}' (${site.url}) not supported`);
 }
 
+async function fetchClassic(site, page) {
+    const res = await bhttp.get(`${site.url}/scenes?page=${page}`);
+
+    if (res.statusCode === 200) {
+        return scrapeLatestClassic(res.body.toString(), site);
+    }
+
+    return null;
+}
+
 async function fetchLatest(site, page = 1) {
+    if (site.parameters && site.parameters.classic) {
+        return fetchClassic(site, page);
+    }
+
     const url = getUrl(site);
     const { search } = new URL(url);
     const siteId = new URLSearchParams(search).get('site');
@@ -129,7 +175,8 @@ async function fetchLatest(site, page = 1) {
 
     const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
     const limit = 10;
-    const res = await session.get(`https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`, {
+    const apiUrl = `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`;
+    const res = await session.get(apiUrl, {
         headers: {
             Instance: instanceToken,
         },

src/sites.js

@@ -62,7 +62,7 @@ async function findSiteByUrl(url) {
             'sites.*',
             'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
         )
-        .where('sites.url', 'like', `%${domain}`)
+        .where('sites.url', 'like', `%${domain}%`)
         .first();
 
     if (site) {