Split 5K from Kelly Madison, added 5KVids network with 5K and 8K sites.
This commit is contained in:
DebaucheryLibrarian
parent
ec58893b81
commit
5f22e9cbdb
|
|
@ -361,6 +361,18 @@ module.exports = {
|
|||
interval: 100,
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.8kmilfs.com': {
|
||||
interval: 1100, // 3 r/s and 60 r/m rate limit
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.pornfidelity.com': {
|
||||
interval: 1100,
|
||||
concurrency: 1,
|
||||
},
|
||||
},
|
||||
apiKeys: {
|
||||
kellyCash: null,
|
||||
fiveKCash: null,
|
||||
},
|
||||
fetchAfter: [1, 'week'],
|
||||
missingDateLimit: 3,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@
|
|||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"csv-stringify": "^6.4.4",
|
||||
"date-fns": "^2.30.0",
|
||||
"dayjs": "^1.11.10",
|
||||
|
|
@ -7967,6 +7968,11 @@
|
|||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz",
|
||||
"integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ=="
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
|
||||
"integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw=="
|
||||
},
|
||||
"node_modules/csv-stringify": {
|
||||
"version": "6.4.4",
|
||||
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.4.tgz",
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@
|
|||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"csv-stringify": "^6.4.4",
|
||||
"date-fns": "^2.30.0",
|
||||
"dayjs": "^1.11.10",
|
||||
|
|
|
|||
|
|
@ -109,6 +109,16 @@ const networks = [
|
|||
},
|
||||
parent: '21sextury',
|
||||
},
|
||||
{
|
||||
slug: '5kvids',
|
||||
name: '5K Vids',
|
||||
url: 'https://www.5kvids.com',
|
||||
parameters: {
|
||||
// layout: 'api',
|
||||
apiKey: 'fiveKCash',
|
||||
apiAddress: 'https://www.8kmilfs.com/api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'adultempire',
|
||||
name: 'Adult Empire',
|
||||
|
|
@ -453,6 +463,11 @@ const networks = [
|
|||
name: 'Kelly Madison Media',
|
||||
url: 'https://www.kellymadison.com',
|
||||
description: 'Home of Kelly Madison and Ryan Madison',
|
||||
parameters: {
|
||||
// layout: 'api',
|
||||
apiKey: 'kellyCash',
|
||||
apiAddress: 'https://www.pornfidelity.com/api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'killergram',
|
||||
|
|
|
|||
|
|
@ -5318,7 +5318,7 @@ const sites = [
|
|||
hasLogo: false,
|
||||
parent: 'karups',
|
||||
},
|
||||
// KELLY MADISON MEDIA
|
||||
// KELLY MADISON MEDIA / 5K / 8K
|
||||
{
|
||||
slug: 'teenfidelity',
|
||||
name: 'Teen Fidelity',
|
||||
|
|
@ -5357,9 +5357,8 @@ const sites = [
|
|||
name: '5Kporn',
|
||||
url: 'https://www.5kporn.com',
|
||||
tags: ['5k'],
|
||||
parent: 'kellymadison',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
// IDs overlap with Fidelity sites
|
||||
siteId: 1,
|
||||
},
|
||||
},
|
||||
|
|
@ -5368,11 +5367,29 @@ const sites = [
|
|||
name: '5Kteens',
|
||||
url: 'https://www.5kteens.com',
|
||||
tags: ['5k'],
|
||||
parent: 'kellymadison',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
siteId: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: '8kmilfs',
|
||||
name: '8Kmilfs',
|
||||
url: 'https://www.8kmilfs.com',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
siteId: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: '8kteens',
|
||||
name: '8Kteens',
|
||||
url: 'https://www.8kteens.com',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
siteId: 4,
|
||||
},
|
||||
},
|
||||
// KILLERGRAM
|
||||
{
|
||||
name: 'Urban Perversions',
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const unprint = require('unprint');
|
||||
const { parse } = require('csv-parse/sync');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const qu = require('../utils/qu');
|
||||
|
|
@ -67,6 +69,45 @@ function scrapeLatest(scenes, site) {
|
|||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
|
||||
const res = await http.get(url, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.body.status === 'success') {
|
||||
return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestApi(channel, page = 1, { parameters }) {
|
||||
// JSON API doesn't return poster images, CSV API doesn't have pagination
|
||||
const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(res.body.data[1]);
|
||||
|
||||
if (res.ok) {
|
||||
const data = parse(res.body, {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
console.log(data);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
|
||||
const { pathname } = new URL(url);
|
||||
const release = {};
|
||||
|
|
@ -128,6 +169,21 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
|
|||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel, baseRelease) {
|
||||
const session = http.session();
|
||||
|
||||
const res = await qu.get(url, null, {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}, {
|
||||
session,
|
||||
followRedirects: false, // redirects to sign-up page if scene not found
|
||||
});
|
||||
|
||||
return res.ok
|
||||
? scrapeScene(res.item, url, baseRelease, channel, session)
|
||||
: res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
|
|
@ -164,36 +220,6 @@ function scrapeProfile({ query }) {
|
|||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
|
||||
const res = await http.get(url, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.body.status === 'success') {
|
||||
return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel, baseRelease) {
|
||||
const session = http.session();
|
||||
|
||||
const res = await qu.get(url, null, {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}, {
|
||||
session,
|
||||
followRedirects: false, // redirects to sign-up page if scene not found
|
||||
});
|
||||
|
||||
return res.ok
|
||||
? scrapeScene(res.item, url, baseRelease, channel, session)
|
||||
: res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, { entity }) {
|
||||
const actorSlug = slugify(actorName);
|
||||
|
||||
|
|
@ -214,4 +240,8 @@ module.exports = {
|
|||
fetchLatest,
|
||||
fetchProfile,
|
||||
fetchScene,
|
||||
api: {
|
||||
fetchLatest: fetchLatestApi,
|
||||
// fetchScene, fetchSceneApi,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ const scrapers = {
|
|||
julesjordan,
|
||||
karups,
|
||||
kellymadison,
|
||||
'5kvids': kellymadison,
|
||||
killergram,
|
||||
kink,
|
||||
kinkvr: badoink,
|
||||
|
|
@ -268,7 +269,7 @@ const scrapers = {
|
|||
julesjordan,
|
||||
karups,
|
||||
kellymadison,
|
||||
'5kporn': kellymadison,
|
||||
'5kvids': kellymadison,
|
||||
killergram,
|
||||
kink,
|
||||
kinkmen: kink,
|
||||
|
|
|
|||
Loading…
Reference in New Issue