Split 5K from Kelly Madison, added 5KVids network with 5K and 8K sites.

config/default.js

@@ -361,6 +361,18 @@ module.exports = {
 			interval: 100,
 			concurrency: 1,
 		},
+		'www.8kmilfs.com': {
+			interval: 1100, // 3 r/s and 60 r/m rate limit
+			concurrency: 1,
+		},
+		'www.pornfidelity.com': {
+			interval: 1100,
+			concurrency: 1,
+		},
+	},
+	apiKeys: {
+		kellyCash: null,
+		fiveKCash: null,
 	},
 	fetchAfter: [1, 'week'],
 	missingDateLimit: 3,

package-lock.json

@@ -34,6 +34,7 @@
                 "connect-session-knex": "^4.0.0",
                 "convert": "^4.14.0",
                 "cookie": "^0.6.0",
+                "csv-parse": "^6.1.0",
                 "csv-stringify": "^6.4.4",
                 "date-fns": "^2.30.0",
                 "dayjs": "^1.11.10",
@@ -7967,6 +7968,11 @@
             "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz",
             "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ=="
         },
+        "node_modules/csv-parse": {
+            "version": "6.1.0",
+            "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
+            "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw=="
+        },
         "node_modules/csv-stringify": {
             "version": "6.4.4",
             "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.4.tgz",

package.json

@@ -93,6 +93,7 @@
         "connect-session-knex": "^4.0.0",
         "convert": "^4.14.0",
         "cookie": "^0.6.0",
+        "csv-parse": "^6.1.0",
         "csv-stringify": "^6.4.4",
         "date-fns": "^2.30.0",
         "dayjs": "^1.11.10",

seeds/01_networks.js

@@ -109,6 +109,16 @@ const networks = [
 		},
 		parent: '21sextury',
 	},
+	{
+		slug: '5kvids',
+		name: '5K Vids',
+		url: 'https://www.5kvids.com',
+		parameters: {
+			// layout: 'api',
+			apiKey: 'fiveKCash',
+			apiAddress: 'https://www.8kmilfs.com/api',
+		},
+	},
 	{
 		slug: 'adultempire',
 		name: 'Adult Empire',
@@ -453,6 +463,11 @@ const networks = [
 		name: 'Kelly Madison Media',
 		url: 'https://www.kellymadison.com',
 		description: 'Home of Kelly Madison and Ryan Madison',
+		parameters: {
+			// layout: 'api',
+			apiKey: 'kellyCash',
+			apiAddress: 'https://www.pornfidelity.com/api',
+		},
 	},
 	{
 		slug: 'killergram',

seeds/02_sites.js

@@ -5318,7 +5318,7 @@ const sites = [
 		hasLogo: false,
 		parent: 'karups',
 	},
-	// KELLY MADISON MEDIA
+	// KELLY MADISON MEDIA / 5K / 8K
 	{
 		slug: 'teenfidelity',
 		name: 'Teen Fidelity',
@@ -5357,9 +5357,8 @@ const sites = [
 		name: '5Kporn',
 		url: 'https://www.5kporn.com',
 		tags: ['5k'],
-		parent: 'kellymadison',
+		parent: '5kvids',
 		parameters: {
-			// IDs overlap with Fidelity sites
 			siteId: 1,
 		},
 	},
@@ -5368,11 +5367,29 @@ const sites = [
 		name: '5Kteens',
 		url: 'https://www.5kteens.com',
 		tags: ['5k'],
-		parent: 'kellymadison',
+		parent: '5kvids',
 		parameters: {
 			siteId: 2,
 		},
 	},
+	{
+		slug: '8kmilfs',
+		name: '8Kmilfs',
+		url: 'https://www.8kmilfs.com',
+		parent: '5kvids',
+		parameters: {
+			siteId: 3,
+		},
+	},
+	{
+		slug: '8kteens',
+		name: '8Kteens',
+		url: 'https://www.8kteens.com',
+		parent: '5kvids',
+		parameters: {
+			siteId: 4,
+		},
+	},
 	// KILLERGRAM
 	{
 		name: 'Urban Perversions',

src/scrapers/kellymadison.js

@@ -1,6 +1,8 @@
 'use strict';
 
+const config = require('config');
 const unprint = require('unprint');
+const { parse } = require('csv-parse/sync');
 
 const slugify = require('../utils/slugify');
 const qu = require('../utils/qu');
@@ -67,6 +69,45 @@ function scrapeLatest(scenes, site) {
 	});
 }
 
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
+	const res = await http.get(url, {
+		headers: {
+			'X-Requested-With': 'XMLHttpRequest',
+		},
+	});
+
+	if (res.ok && res.body.status === 'success') {
+		return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestApi(channel, page = 1, { parameters }) {
+	// JSON API doesn't return poster images, CSV API doesn't have pagination
+	const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
+		headers: {
+			Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`,
+		},
+	});
+
+	console.log(res.body.data[1]);
+
+	if (res.ok) {
+		const data = parse(res.body, {
+			columns: true,
+			skip_empty_lines: true,
+		});
+
+		console.log(data);
+
+		return null;
+	}
+
+	return res.status;
+}
+
 async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
 	const { pathname } = new URL(url);
 	const release = {};
@@ -128,6 +169,21 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
 	return release;
 }
 
+async function fetchScene(url, channel, baseRelease) {
+	const session = http.session();
+
+	const res = await qu.get(url, null, {
+		'X-Requested-With': 'XMLHttpRequest',
+	}, {
+		session,
+		followRedirects: false, // redirects to sign-up page if scene not found
+	});
+
+	return res.ok
+		? scrapeScene(res.item, url, baseRelease, channel, session)
+		: res.status;
+}
+
 function scrapeProfile({ query }) {
 	const profile = {};
 
@@ -164,36 +220,6 @@ function scrapeProfile({ query }) {
 	return profile;
 }
 
-async function fetchLatest(channel, page = 1) {
-	const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
-	const res = await http.get(url, {
-		headers: {
-			'X-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (res.ok && res.body.status === 'success') {
-		return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
-	}
-
-	return res.status;
-}
-
-async function fetchScene(url, channel, baseRelease) {
-	const session = http.session();
-
-	const res = await qu.get(url, null, {
-		'X-Requested-With': 'XMLHttpRequest',
-	}, {
-		session,
-		followRedirects: false, // redirects to sign-up page if scene not found
-	});
-
-	return res.ok
-		? scrapeScene(res.item, url, baseRelease, channel, session)
-		: res.status;
-}
-
 async function fetchProfile({ name: actorName }, { entity }) {
 	const actorSlug = slugify(actorName);
 
@@ -214,4 +240,8 @@ module.exports = {
 	fetchLatest,
 	fetchProfile,
 	fetchScene,
+	api: {
+		fetchLatest: fetchLatestApi,
+		// fetchScene, fetchSceneApi,
+	},
 };

src/scrapers/scrapers.js

@@ -135,6 +135,7 @@ const scrapers = {
 		julesjordan,
 		karups,
 		kellymadison,
+		'5kvids': kellymadison,
 		killergram,
 		kink,
 		kinkvr: badoink,
@@ -268,7 +269,7 @@ const scrapers = {
 		julesjordan,
 		karups,
 		kellymadison,
-		'5kporn': kellymadison,
+		'5kvids': kellymadison,
 		killergram,
 		kink,
 		kinkmen: kink,