Split 5K from Kelly Madison, added 5KVids network with 5K and 8K sites.

src/scrapers/kellymadison.js

@@ -1,6 +1,8 @@
 'use strict';
 
+const config = require('config');
 const unprint = require('unprint');
+const { parse } = require('csv-parse/sync');
 
 const slugify = require('../utils/slugify');
 const qu = require('../utils/qu');
@@ -67,6 +69,45 @@ function scrapeLatest(scenes, site) {
 	});
 }
 
+async function fetchLatest(channel, page = 1) {
+	const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
+	const res = await http.get(url, {
+		headers: {
+			'X-Requested-With': 'XMLHttpRequest',
+		},
+	});
+
+	if (res.ok && res.body.status === 'success') {
+		return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
+	}
+
+	return res.status;
+}
+
+async function fetchLatestApi(channel, page = 1, { parameters }) {
+	// JSON API doesn't return poster images, CSV API doesn't have pagination
+	const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
+		headers: {
+			Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`,
+		},
+	});
+
+	console.log(res.body.data[1]);
+
+	if (res.ok) {
+		const data = parse(res.body, {
+			columns: true,
+			skip_empty_lines: true,
+		});
+
+		console.log(data);
+
+		return null;
+	}
+
+	return res.status;
+}
+
 async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
 	const { pathname } = new URL(url);
 	const release = {};
@@ -128,6 +169,21 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
 	return release;
 }
 
+async function fetchScene(url, channel, baseRelease) {
+	const session = http.session();
+
+	const res = await qu.get(url, null, {
+		'X-Requested-With': 'XMLHttpRequest',
+	}, {
+		session,
+		followRedirects: false, // redirects to sign-up page if scene not found
+	});
+
+	return res.ok
+		? scrapeScene(res.item, url, baseRelease, channel, session)
+		: res.status;
+}
+
 function scrapeProfile({ query }) {
 	const profile = {};
 
@@ -164,36 +220,6 @@ function scrapeProfile({ query }) {
 	return profile;
 }
 
-async function fetchLatest(channel, page = 1) {
-	const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
-	const res = await http.get(url, {
-		headers: {
-			'X-Requested-With': 'XMLHttpRequest',
-		},
-	});
-
-	if (res.ok && res.body.status === 'success') {
-		return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
-	}
-
-	return res.status;
-}
-
-async function fetchScene(url, channel, baseRelease) {
-	const session = http.session();
-
-	const res = await qu.get(url, null, {
-		'X-Requested-With': 'XMLHttpRequest',
-	}, {
-		session,
-		followRedirects: false, // redirects to sign-up page if scene not found
-	});
-
-	return res.ok
-		? scrapeScene(res.item, url, baseRelease, channel, session)
-		: res.status;
-}
-
 async function fetchProfile({ name: actorName }, { entity }) {
 	const actorSlug = slugify(actorName);
 
@@ -214,4 +240,8 @@ module.exports = {
 	fetchLatest,
 	fetchProfile,
 	fetchScene,
+	api: {
+		fetchLatest: fetchLatestApi,
+		// fetchScene, fetchSceneApi,
+	},
 };

src/scrapers/scrapers.js

@@ -135,6 +135,7 @@ const scrapers = {
 		julesjordan,
 		karups,
 		kellymadison,
+		'5kvids': kellymadison,
 		killergram,
 		kink,
 		kinkvr: badoink,
@@ -268,7 +269,7 @@ const scrapers = {
 		julesjordan,
 		karups,
 		kellymadison,
-		'5kporn': kellymadison,
+		'5kvids': kellymadison,
 		killergram,
 		kink,
 		kinkmen: kink,