From 5f22e9cbdb2f09c1ae51b4c68b855fc0acc3a1a0 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 21 Nov 2025 04:47:42 +0100 Subject: [PATCH] Split 5K from Kelly Madison, added 5KVids network with 5K and 8K sites. --- config/default.js | 12 +++++ package-lock.json | 6 +++ package.json | 1 + seeds/01_networks.js | 15 ++++++ seeds/02_sites.js | 25 ++++++++-- src/scrapers/kellymadison.js | 90 ++++++++++++++++++++++++------------ src/scrapers/scrapers.js | 3 +- 7 files changed, 117 insertions(+), 35 deletions(-) diff --git a/config/default.js b/config/default.js index 7a0a1cf0..beddffe4 100755 --- a/config/default.js +++ b/config/default.js @@ -361,6 +361,18 @@ module.exports = { interval: 100, concurrency: 1, }, + 'www.8kmilfs.com': { + interval: 1100, // 3 r/s and 60 r/m rate limit + concurrency: 1, + }, + 'www.pornfidelity.com': { + interval: 1100, + concurrency: 1, + }, + }, + apiKeys: { + kellyCash: null, + fiveKCash: null, }, fetchAfter: [1, 'week'], missingDateLimit: 3, diff --git a/package-lock.json b/package-lock.json index 20822d22..0502d724 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "connect-session-knex": "^4.0.0", "convert": "^4.14.0", "cookie": "^0.6.0", + "csv-parse": "^6.1.0", "csv-stringify": "^6.4.4", "date-fns": "^2.30.0", "dayjs": "^1.11.10", @@ -7967,6 +7968,11 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" }, + "node_modules/csv-parse": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz", + "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==" + }, "node_modules/csv-stringify": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.4.tgz", diff --git a/package.json b/package.json index e69e8fb4..caf60fdb 100755 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "connect-session-knex": "^4.0.0", "convert": "^4.14.0", "cookie": "^0.6.0", + "csv-parse": "^6.1.0", "csv-stringify": "^6.4.4", "date-fns": "^2.30.0", "dayjs": "^1.11.10", diff --git a/seeds/01_networks.js b/seeds/01_networks.js index d0100451..3f40614a 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -109,6 +109,16 @@ const networks = [ }, parent: '21sextury', }, + { + slug: '5kvids', + name: '5K Vids', + url: 'https://www.5kvids.com', + parameters: { + // layout: 'api', + apiKey: 'fiveKCash', + apiAddress: 'https://www.8kmilfs.com/api', + }, + }, { slug: 'adultempire', name: 'Adult Empire', @@ -453,6 +463,11 @@ const networks = [ name: 'Kelly Madison Media', url: 'https://www.kellymadison.com', description: 'Home of Kelly Madison and Ryan Madison', + parameters: { + // layout: 'api', + apiKey: 'kellyCash', + apiAddress: 'https://www.pornfidelity.com/api', + }, }, { slug: 'killergram', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index b837bb95..ec039937 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -5318,7 +5318,7 @@ const sites = [ hasLogo: false, parent: 'karups', }, - // KELLY MADISON MEDIA + // KELLY MADISON MEDIA / 5K / 8K { slug: 'teenfidelity', name: 'Teen Fidelity', @@ -5357,9 +5357,8 @@ const sites = [ name: '5Kporn', url: 'https://www.5kporn.com', tags: ['5k'], - parent: 'kellymadison', + parent: '5kvids', parameters: { - // IDs overlap with Fidelity sites siteId: 1, }, }, @@ -5368,11 +5367,29 @@ const sites = [ name: '5Kteens', url: 'https://www.5kteens.com', tags: ['5k'], - parent: 'kellymadison', + parent: '5kvids', parameters: { siteId: 2, }, }, + { + slug: '8kmilfs', + name: '8Kmilfs', + url: 'https://www.8kmilfs.com', + parent: '5kvids', + parameters: { + siteId: 3, + }, + }, + { + slug: '8kteens', + name: '8Kteens', + url: 'https://www.8kteens.com', + parent: '5kvids', + parameters: { + siteId: 4, + }, + }, // KILLERGRAM { name: 'Urban Perversions', diff --git a/src/scrapers/kellymadison.js b/src/scrapers/kellymadison.js index ca5b220e..75bec43d 100755 --- a/src/scrapers/kellymadison.js +++ b/src/scrapers/kellymadison.js @@ -1,6 +1,8 @@ 'use strict'; +const config = require('config'); const unprint = require('unprint'); +const { parse } = require('csv-parse/sync'); const slugify = require('../utils/slugify'); const qu = require('../utils/qu'); @@ -67,6 +69,45 @@ function scrapeLatest(scenes, site) { }); } +async function fetchLatest(channel, page = 1) { + const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites + const res = await http.get(url, { + headers: { + 'X-Requested-With': 'XMLHttpRequest', + }, + }); + + if (res.ok && res.body.status === 'success') { + return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel); + } + + return res.status; +} + +async function fetchLatestApi(channel, page = 1, { parameters }) { + // JSON API doesn't return poster images, CSV API doesn't have pagination + const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, { + headers: { + Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`, + }, + }); + + console.log(res.body.data[1]); + + if (res.ok) { + const data = parse(res.body, { + columns: true, + skip_empty_lines: true, + }); + + console.log(data); + + return null; + } + + return res.status; +} + async function scrapeScene({ query, html }, url, baseRelease, channel, session) { const { pathname } = new URL(url); const release = {}; @@ -128,6 +169,21 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session) return release; } +async function fetchScene(url, channel, baseRelease) { + const session = http.session(); + + const res = await qu.get(url, null, { + 'X-Requested-With': 'XMLHttpRequest', + }, { + session, + followRedirects: false, // redirects to sign-up page if scene not found + }); + + return res.ok + ? scrapeScene(res.item, url, baseRelease, channel, session) + : res.status; +} + function scrapeProfile({ query }) { const profile = {}; @@ -164,36 +220,6 @@ function scrapeProfile({ query }) { return profile; } -async function fetchLatest(channel, page = 1) { - const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites - const res = await http.get(url, { - headers: { - 'X-Requested-With': 'XMLHttpRequest', - }, - }); - - if (res.ok && res.body.status === 'success') { - return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel); - } - - return res.status; -} - -async function fetchScene(url, channel, baseRelease) { - const session = http.session(); - - const res = await qu.get(url, null, { - 'X-Requested-With': 'XMLHttpRequest', - }, { - session, - followRedirects: false, // redirects to sign-up page if scene not found - }); - - return res.ok - ? scrapeScene(res.item, url, baseRelease, channel, session) - : res.status; -} - async function fetchProfile({ name: actorName }, { entity }) { const actorSlug = slugify(actorName); @@ -214,4 +240,8 @@ module.exports = { fetchLatest, fetchProfile, fetchScene, + api: { + fetchLatest: fetchLatestApi, + // fetchScene, fetchSceneApi, + }, }; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 77f9d261..eb9c2b05 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -135,6 +135,7 @@ const scrapers = { julesjordan, karups, kellymadison, + '5kvids': kellymadison, killergram, kink, kinkvr: badoink, @@ -268,7 +269,7 @@ const scrapers = { julesjordan, karups, kellymadison, - '5kporn': kellymadison, + '5kvids': kellymadison, killergram, kink, kinkmen: kink,