Compare commits
No commits in common. "bd7f8b69b00b2afa564e707834a95d6797e1db6c" and "ec58893b817151d8be9d9bfa0120d56034097924" have entirely different histories.
bd7f8b69b0
...
ec58893b81
|
|
@ -361,18 +361,6 @@ module.exports = {
|
|||
interval: 100,
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.8kmilfs.com': {
|
||||
interval: 1100, // 3 r/s and 60 r/m rate limit
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.pornfidelity.com': {
|
||||
interval: 1100,
|
||||
concurrency: 1,
|
||||
},
|
||||
},
|
||||
apiKeys: {
|
||||
kellyCash: null,
|
||||
fiveKCash: null,
|
||||
},
|
||||
fetchAfter: [1, 'week'],
|
||||
missingDateLimit: 3,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.244.77",
|
||||
"version": "1.244.76",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "traxxx",
|
||||
"version": "1.244.77",
|
||||
"version": "1.244.76",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
|
|
@ -34,7 +34,6 @@
|
|||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"csv-stringify": "^6.4.4",
|
||||
"date-fns": "^2.30.0",
|
||||
"dayjs": "^1.11.10",
|
||||
|
|
@ -7968,11 +7967,6 @@
|
|||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz",
|
||||
"integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ=="
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
|
||||
"integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw=="
|
||||
},
|
||||
"node_modules/csv-stringify": {
|
||||
"version": "6.4.4",
|
||||
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.4.tgz",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.244.77",
|
||||
"version": "1.244.76",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"scripts": {
|
||||
|
|
@ -93,7 +93,6 @@
|
|||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"csv-stringify": "^6.4.4",
|
||||
"date-fns": "^2.30.0",
|
||||
"dayjs": "^1.11.10",
|
||||
|
|
|
|||
|
|
@ -109,16 +109,6 @@ const networks = [
|
|||
},
|
||||
parent: '21sextury',
|
||||
},
|
||||
{
|
||||
slug: '5kvids',
|
||||
name: '5K Vids',
|
||||
url: 'https://www.5kvids.com',
|
||||
parameters: {
|
||||
// layout: 'api',
|
||||
apiKey: 'fiveKCash',
|
||||
apiAddress: 'https://www.8kmilfs.com/api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'adultempire',
|
||||
name: 'Adult Empire',
|
||||
|
|
@ -463,11 +453,6 @@ const networks = [
|
|||
name: 'Kelly Madison Media',
|
||||
url: 'https://www.kellymadison.com',
|
||||
description: 'Home of Kelly Madison and Ryan Madison',
|
||||
parameters: {
|
||||
// layout: 'api',
|
||||
apiKey: 'kellyCash',
|
||||
apiAddress: 'https://www.pornfidelity.com/api',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'killergram',
|
||||
|
|
|
|||
|
|
@ -5318,7 +5318,7 @@ const sites = [
|
|||
hasLogo: false,
|
||||
parent: 'karups',
|
||||
},
|
||||
// KELLY MADISON MEDIA / 5K / 8K
|
||||
// KELLY MADISON MEDIA
|
||||
{
|
||||
slug: 'teenfidelity',
|
||||
name: 'Teen Fidelity',
|
||||
|
|
@ -5357,8 +5357,9 @@ const sites = [
|
|||
name: '5Kporn',
|
||||
url: 'https://www.5kporn.com',
|
||||
tags: ['5k'],
|
||||
parent: '5kvids',
|
||||
parent: 'kellymadison',
|
||||
parameters: {
|
||||
// IDs overlap with Fidelity sites
|
||||
siteId: 1,
|
||||
},
|
||||
},
|
||||
|
|
@ -5367,29 +5368,11 @@ const sites = [
|
|||
name: '5Kteens',
|
||||
url: 'https://www.5kteens.com',
|
||||
tags: ['5k'],
|
||||
parent: '5kvids',
|
||||
parent: 'kellymadison',
|
||||
parameters: {
|
||||
siteId: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: '8kmilfs',
|
||||
name: '8Kmilfs',
|
||||
url: 'https://www.8kmilfs.com',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
siteId: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: '8kteens',
|
||||
name: '8Kteens',
|
||||
url: 'https://www.8kteens.com',
|
||||
parent: '5kvids',
|
||||
parameters: {
|
||||
siteId: 4,
|
||||
},
|
||||
},
|
||||
// KILLERGRAM
|
||||
{
|
||||
name: 'Urban Perversions',
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const unprint = require('unprint');
|
||||
const { parse } = require('csv-parse/sync');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const qu = require('../utils/qu');
|
||||
|
|
@ -69,45 +67,6 @@ function scrapeLatest(scenes, site) {
|
|||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
|
||||
const res = await http.get(url, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.body.status === 'success') {
|
||||
return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestApi(channel, page = 1, { parameters }) {
|
||||
// JSON API doesn't return poster images, CSV API doesn't have pagination
|
||||
const res = await http.get(`${parameters.apiAddress}/affiliates?site_id=${parameters.siteId}&page=${page}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.apiKeys[parameters.apiKey]}`,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(res.body.data[1]);
|
||||
|
||||
if (res.ok) {
|
||||
const data = parse(res.body, {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
});
|
||||
|
||||
console.log(data);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query, html }, url, baseRelease, channel, session) {
|
||||
const { pathname } = new URL(url);
|
||||
const release = {};
|
||||
|
|
@ -169,21 +128,6 @@ async function scrapeScene({ query, html }, url, baseRelease, channel, session)
|
|||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel, baseRelease) {
|
||||
const session = http.session();
|
||||
|
||||
const res = await qu.get(url, null, {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}, {
|
||||
session,
|
||||
followRedirects: false, // redirects to sign-up page if scene not found
|
||||
});
|
||||
|
||||
return res.ok
|
||||
? scrapeScene(res.item, url, baseRelease, channel, session)
|
||||
: res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
|
|
@ -220,6 +164,36 @@ function scrapeProfile({ query }) {
|
|||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/episodes/search?page=${page}&site=${channel.parameters.siteId || ''}`; // TLS issues with teenfidelity.com, same overview on all sites
|
||||
const res = await http.get(url, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.body.status === 'success') {
|
||||
return scrapeLatest(qu.extractAll(res.body.html, '.episode, .ep'), channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel, baseRelease) {
|
||||
const session = http.session();
|
||||
|
||||
const res = await qu.get(url, null, {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
}, {
|
||||
session,
|
||||
followRedirects: false, // redirects to sign-up page if scene not found
|
||||
});
|
||||
|
||||
return res.ok
|
||||
? scrapeScene(res.item, url, baseRelease, channel, session)
|
||||
: res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }, { entity }) {
|
||||
const actorSlug = slugify(actorName);
|
||||
|
||||
|
|
@ -240,8 +214,4 @@ module.exports = {
|
|||
fetchLatest,
|
||||
fetchProfile,
|
||||
fetchScene,
|
||||
api: {
|
||||
fetchLatest: fetchLatestApi,
|
||||
// fetchScene, fetchSceneApi,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -135,7 +135,6 @@ const scrapers = {
|
|||
julesjordan,
|
||||
karups,
|
||||
kellymadison,
|
||||
'5kvids': kellymadison,
|
||||
killergram,
|
||||
kink,
|
||||
kinkvr: badoink,
|
||||
|
|
@ -269,7 +268,7 @@ const scrapers = {
|
|||
julesjordan,
|
||||
karups,
|
||||
kellymadison,
|
||||
'5kvids': kellymadison,
|
||||
'5kporn': kellymadison,
|
||||
killergram,
|
||||
kink,
|
||||
kinkmen: kink,
|
||||
|
|
|
|||
Loading…
Reference in New Issue