Added Adult Time. Adding context to logger.

src/scrapers/adulttime.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
+
+function curateRelease(release, site) {
+    if (['bubblegumdungeon', 'ladygonzo'].includes(site.slug)) {
+        return {
+            ...release,
+            title: release.title.split(/:|\|/)[1].trim(),
+        };
+    }
+
+    return release;
+}
+
+async function networkFetchScene(url, site) {
+    const scene = await fetchScene(url, site);
+
+    return curateRelease(scene, site);
+}
+
+async function fetchLatest(site, page = 1) {
+    const releases = await fetchApiLatest(site, page, false);
+
+    console.log(releases);
+
+    return releases.map(release => curateRelease(release, site));
+}
+
+async function fetchUpcoming(site, page = 1) {
+    const releases = await fetchApiUpcoming(site, page, false);
+
+    return releases.map(release => curateRelease(release, site));
+}
+
+module.exports = {
+    fetchLatest,
+    fetchProfile: fetchApiProfile,
+    fetchScene: networkFetchScene,
+    fetchUpcoming,
+};

src/scrapers/gamma.js

@@ -7,6 +7,7 @@ const cheerio = require('cheerio');
 const moment = require('moment');
 
 const argv = require('../argv');
+const logger = require('../logger');
 const { ex, get } = require('../utils/q');
 const slugify = require('../utils/slugify');
 
@@ -71,7 +72,7 @@ async function getPhotos(albumPath, site) {
 
         return photos;
     } catch (error) {
-        console.error(`Failed to fetch ${site.name} photos from ${albumUrl}: ${error.message}`);
+        logger.warn(`Failed to fetch ${site.name} photos from ${albumUrl}: ${error.message}`);
 
         return [];
     }
@@ -88,11 +89,16 @@ async function scrapeApiReleases(json, site) {
             dislikes: scene.ratings_down,
         };
 
-        release.url = `${site.url}/en/video/${scene.url_title}/${release.entryId}`;
+        release.url = site.parameters?.scene
+            ? `${site.parameters.scene}/${scene.url_title}/${release.entryId}`
+            : `${site.url}/en/video/${scene.url_title}/${release.entryId}`;
+
         release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
         release.actors = scene.actors.map(({ name }) => name);
         release.director = scene.directors[0].name;
 
+        console.log(release.url);
+
         release.tags = scene.master_categories.concat(scene.categories?.map(category => category.name));
 
         const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]);
@@ -159,7 +165,7 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) {
 
 async function scrapeScene(html, url, site) {
     const $ = cheerio.load(html, { normalizeWhitespace: true });
-    const release = { $ };
+    const release = { $, url };
 
     const json = $('script[type="application/ld+json"]').html();
     const videoJson = $('script:contains("window.ScenePlayerOptions")').html();
@@ -325,20 +331,7 @@ function scrapeApiProfile(data, releases, siteSlug) {
     return profile;
 }
 
-async function fetchApiCredentials(referer) {
-    const res = await bhttp.get(referer);
-    const body = res.body.toString();
-
-    const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
-
-    if (!apiLine) {
-        throw new Error(`Can not use Gamma API for ${referer}`);
-    }
-
-    const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
-    const apiData = JSON.parse(apiSerial);
-
-    const { applicationID: appId, apiKey } = apiData.api.algolia;
+function getApiUrl(appId, apiKey) {
     const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0';
 
     const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
@@ -351,9 +344,31 @@ async function fetchApiCredentials(referer) {
     };
 }
 
+async function fetchApiCredentials(referer, site) {
+    if (site.parameters?.appId && site.parameters?.apiKey) {
+        return getApiUrl(site.parameters.appId, site.parameters.apiKey);
+    }
+
+    const res = await bhttp.get(referer);
+    const body = res.body.toString();
+
+    const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey'));
+
+    if (!apiLine) {
+        throw new Error(`No Gamma API key found for ${referer}`);
+    }
+
+    const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1);
+    const apiData = JSON.parse(apiSerial);
+
+    const { applicationID: appId, apiKey } = apiData.api.algolia;
+
+    return getApiUrl(appId, apiKey);
+}
+
 async function fetchApiLatest(site, page = 1, upcoming = false) {
-    const referer = `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`;
-    const { apiUrl } = await fetchApiCredentials(referer);
+    const referer = site.parameters?.referer || `${site.parameters?.networkReferer ? site.network.url : site.url}/en/videos`;
+    const { apiUrl } = await fetchApiCredentials(referer, site);
 
     const res = await bhttp.post(apiUrl, {
         requests: [
@@ -394,7 +409,11 @@ async function fetchUpcoming(site) {
     return scrapeAll(res.body.toString(), site);
 }
 
-async function fetchScene(url, site) {
+async function fetchScene(url, site, release) {
+    if (site.parameters?.deep === false) {
+        return release;
+    }
+
     const res = await bhttp.get(url);
 
     return scrapeScene(res.body.toString(), url, site);

src/scrapers/scrapers.js

@@ -1,6 +1,7 @@
 'use strict';
 
 // releases
+const adulttime = require('./adulttime');
 const babes = require('./babes');
 const bang = require('./bang');
 const dogfart = require('./dogfart');
@@ -51,6 +52,7 @@ const pornhub = require('./pornhub');
 
 module.exports = {
     releases: {
+        adulttime,
         '21sextury': twentyonesextury,
         babes,
         bang,