Bumped Node to 26 LTS, replaced deprecated dependencies. Switched Nubiles from browser to request interface, seems to fare better with security wall.
This commit is contained in:
DebaucheryLibrarian
Generated
+20
-1510
File diff suppressed because it is too large
Load Diff
+5
-3
@@ -3,6 +3,9 @@
|
||||
"version": "1.256.5",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"engines": {
|
||||
"node": ">=26"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node -r source-map-support/register src/init.js",
|
||||
"webpack": "webpack --env=production --mode=production",
|
||||
@@ -58,7 +61,6 @@
|
||||
"eslint-watch": "^8.0.0",
|
||||
"eslint-webpack-plugin": "^4.0.1",
|
||||
"mini-css-extract-plugin": "^2.7.6",
|
||||
"node-sass": "^9.0.0",
|
||||
"postcss-loader": "^7.3.3",
|
||||
"raw-loader": "^4.0.2",
|
||||
"sass-loader": "^13.3.2",
|
||||
@@ -76,6 +78,7 @@
|
||||
"@graphile-contrib/pg-order-by-related": "^1.0.0",
|
||||
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
||||
"@graphile/pg-aggregates": "^0.1.1",
|
||||
"@noble/hashes": "^1.8.0",
|
||||
"@rsc-parser/react-client": "^1.1.2",
|
||||
"@tensorflow/tfjs-node": "^4.22.0",
|
||||
"@vladmandic/human": "^3.3.6",
|
||||
@@ -83,7 +86,6 @@
|
||||
"array-equal": "^1.0.2",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"bhttp": "^1.2.8",
|
||||
"blake2": "^5.0.0",
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
@@ -92,7 +94,7 @@
|
||||
"cli-confirm": "^1.0.1",
|
||||
"clipboardy": "^2.3.0",
|
||||
"cloudscraper": "^4.6.0",
|
||||
"config": "^3.3.9",
|
||||
"config": "^3.3.12",
|
||||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
|
||||
+5
-20
@@ -6027,56 +6027,41 @@ const sites = [
|
||||
slug: 'sexuallybroken',
|
||||
name: 'Sexually Broken',
|
||||
alias: ['seb'],
|
||||
url: 'https://www.sexuallybroken.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=sexuallybroken.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.sexuallybroken.com/sb',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'infernalrestraints',
|
||||
name: 'Infernal Restraints',
|
||||
alias: ['infr'],
|
||||
url: 'https://www.infernalrestraints.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=infernalrestraints.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.infernalrestraints.com/ir',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'hardtied',
|
||||
name: 'Hardtied',
|
||||
alias: ['ht'],
|
||||
url: 'https://www.hardtied.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=hardtied.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.hardtied.com/ht',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'realtimebondage',
|
||||
name: 'Real Time Bondage',
|
||||
alias: ['rtb'],
|
||||
url: 'https://www.realtimebondage.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=realtimebondage.com',
|
||||
tags: ['bdsm', 'live'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.realtimebondage.com/rtb',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'topgrl',
|
||||
name: 'TopGrl',
|
||||
alias: ['tg'],
|
||||
url: 'https://www.topgrl.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=topgrl.com',
|
||||
tags: ['bdsm', 'femdom'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.topgrl.com/tg',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'paintoy',
|
||||
|
||||
+2
-6
@@ -4,7 +4,7 @@ const config = require('config');
|
||||
const util = require('util');
|
||||
const Promise = require('bluebird');
|
||||
const moment = require('moment');
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
const DOMPurify = require('dompurify');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const omit = require('object.omit');
|
||||
@@ -413,11 +413,7 @@ async function curateProfile(profile, actor) {
|
||||
|
||||
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
|
||||
|
||||
const hasher = curatedProfile.description && blake2
|
||||
.createHash('blake2b', { digestLength: 24 })
|
||||
.update(Buffer.from(slugify(curatedProfile.description)));
|
||||
|
||||
curatedProfile.descriptionHash = curatedProfile.description && hasher.digest('hex');
|
||||
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(slugify(curatedProfile.description), { dkLen: 24 })).toString('hex');
|
||||
|
||||
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
||||
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ const unprintOptions = {
|
||||
logErrors: false,
|
||||
timeout: argv.requestTimeout,
|
||||
userAgent: 'traxxx',
|
||||
browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
// browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
apiUserAgent: 'traxxx',
|
||||
clientRetirement: config.bypass.browser.clientRetirement,
|
||||
remote: config.bypass.browser.remote,
|
||||
|
||||
+7
-1
@@ -3,7 +3,13 @@
|
||||
const config = require('config');
|
||||
const path = require('path');
|
||||
const fsPromises = require('fs').promises;
|
||||
const Human = require('@vladmandic/human').default;
|
||||
const util = require('util');
|
||||
|
||||
// tfjs-node still uses util helpers that were removed (isNullOrUndefined) or deprecated with a runtime warning (isArray) in Node 23+, must be set before it loads
|
||||
util.isNullOrUndefined ??= (value) => value === null || value === undefined;
|
||||
util.isArray = Array.isArray;
|
||||
|
||||
const Human = require('@vladmandic/human').default; // eslint-disable-line import/order
|
||||
|
||||
const knex = require('./knex');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
+3
-3
@@ -12,7 +12,7 @@ const mime = require('mime');
|
||||
// const fileType = require('file-type');
|
||||
const ffmpeg = require('fluent-ffmpeg');
|
||||
const sharp = require('sharp');
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
const { format } = require('date-fns');
|
||||
const taskQueue = require('promise-task-queue');
|
||||
const unprint = require('unprint');
|
||||
@@ -723,7 +723,7 @@ async function fetchSource(source, baseMedia) {
|
||||
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
|
||||
let tempFileTarget;
|
||||
|
||||
const hasher = blake2.createHash('blake2b', { digestLength: 24 });
|
||||
const hasher = blake2b.create({ dkLen: 24 });
|
||||
|
||||
let size = 0;
|
||||
|
||||
@@ -743,7 +743,7 @@ async function fetchSource(source, baseMedia) {
|
||||
? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
|
||||
: await fetchHttpSource(source, tempFileTarget, hashStream);
|
||||
|
||||
const hash = hasher.digest('hex');
|
||||
const hash = Buffer.from(hasher.digest()).toString('hex');
|
||||
|
||||
const [type, subtype] = mimetype.split('/');
|
||||
const extension = mime.getExtension(mimetype);
|
||||
|
||||
+19
-95
@@ -21,51 +21,33 @@ function scrapeLatest(scenes, site) {
|
||||
|
||||
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeLatestLegacy(scenes, site) {
|
||||
return scenes.map(({ query }) => {
|
||||
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
|
||||
const release = {};
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const domain = new URL(site.url).searchParams.get('d');
|
||||
|
||||
const titleEl = query.element('.galleryTitleText, .articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
|
||||
if (!domain) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
const url = `${site.origin}/is/home.php?o=newest&d=${domain}&p=${page}`;
|
||||
|
||||
if (date) {
|
||||
release.title = title.trim();
|
||||
release.date = date;
|
||||
} else {
|
||||
// title should contain date instead, not applicable in brief mode
|
||||
release.title = title.slice(title.indexOf(':') + 1).trim();
|
||||
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
||||
}
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
|
||||
const description = query.content('.articleCopyText');
|
||||
if (description) release.description = description.slice(0, description.lastIndexOf('('));
|
||||
|
||||
const duration = query.duration('.articleCopyText a:nth-child(2)');
|
||||
if (duration) release.duration = duration;
|
||||
|
||||
release.likes = query.number('.articlePostDateText td:nth-child(3)');
|
||||
|
||||
const cover = query.img('a img', { origin: site.url });
|
||||
|
||||
release.covers = [[
|
||||
cover.replace('_thumbnail', ''),
|
||||
cover,
|
||||
]];
|
||||
|
||||
return release;
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: '.column.is-3-desktop',
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
// (UNSEXUALLY) BROKEN, abandoned sites, low priority
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, channel, include, cookies) {
|
||||
@@ -97,60 +79,6 @@ async function scrapeScene({ query }, channel, include, cookies) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeSceneLegacy({ query }, { entity: site }) {
|
||||
const release = {};
|
||||
|
||||
const titleEl = query.element('.articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
|
||||
release.title = title.trim();
|
||||
release.description = query.content('.articleCopyText');
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
|
||||
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
|
||||
|
||||
const [cover, ...photos] = query.imgs('img[src*="images"]');
|
||||
release.covers = [cover];
|
||||
release.photos = photos;
|
||||
|
||||
release.poster = query.poster();
|
||||
|
||||
const trailer = query.video('source');
|
||||
if (trailer) release.trailer = { src: trailer };
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: 'body > .columns .column',
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestLegacy(site, page = 1) {
|
||||
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
|
||||
const res = await unprint.get(url); // JSON containing html as a property
|
||||
|
||||
if (res.ok && res.data?.html) {
|
||||
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease, include) {
|
||||
const res = await unprint.get(url, {
|
||||
cookies: { consent: 'yes' },
|
||||
@@ -166,8 +94,4 @@ async function fetchScene(url, site, baseRelease, include) {
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
legacy: {
|
||||
fetchLatest: fetchLatestLegacy,
|
||||
scrapeScene: scrapeSceneLegacy,
|
||||
},
|
||||
};
|
||||
|
||||
+15
-6
@@ -22,8 +22,9 @@ function stripQuery(link) {
|
||||
|
||||
async function getPhotos(albumUrl) {
|
||||
// aggressive Turnstile protection calls for browser
|
||||
const res = await unprint.browser(albumUrl, {
|
||||
const res = await unprint.get(albumUrl, {
|
||||
selectAll: '.photo-thumb',
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
@@ -73,7 +74,8 @@ function scrapeAll(scenes, entity) {
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
|
||||
|
||||
const res = await unprint.browser(url, {
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
selectAll: '.content-grid-item',
|
||||
});
|
||||
|
||||
@@ -88,7 +90,8 @@ async function fetchUpcoming(site) {
|
||||
if (site.parameters?.upcoming) {
|
||||
const url = `${site.url}/video/upcoming`;
|
||||
|
||||
const res = await unprint.browser(url, {
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
selectAll: '.content-grid-item',
|
||||
});
|
||||
|
||||
@@ -140,7 +143,9 @@ async function scrapeScene({ query }, { url, entity, include }) {
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity, _baseRelease, include) {
|
||||
const res = await unprint.browser(url);
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, { url, entity, include });
|
||||
@@ -184,7 +189,9 @@ async function findModel(actor, entity) {
|
||||
|
||||
const url = `${origin}/model/alpha/${firstLetter}`;
|
||||
|
||||
const resModels = await unprint.browser(url);
|
||||
const resModels = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (!resModels.ok) {
|
||||
return resModels.status;
|
||||
@@ -214,7 +221,9 @@ async function fetchProfile(actor, { entity }) {
|
||||
const model = await findModel(actor, entity);
|
||||
|
||||
if (model) {
|
||||
const resModel = await unprint.browser(model.url);
|
||||
const resModel = await unprint.get(model.url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (resModel.ok) {
|
||||
return scrapeProfile(resModel.context, model.avatar);
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
function getHash(identifier) {
|
||||
const hash = blake2.createHash('blake2b', { digestLength: 8 });
|
||||
|
||||
hash.update(Buffer.from(identifier));
|
||||
|
||||
return hash.digest('hex');
|
||||
return Buffer.from(blake2b(identifier, { dkLen: 8 })).toString('hex');
|
||||
}
|
||||
|
||||
function extractMaleModelsFromTags(tagContainer) {
|
||||
|
||||
+3
-3
@@ -5,7 +5,7 @@ const Promise = require('bluebird');
|
||||
const fsPromises = require('fs').promises;
|
||||
const fs = require('fs');
|
||||
const { PassThrough } = require('stream');
|
||||
const blake2 = require('blake2');
|
||||
const crypto = require('crypto');
|
||||
const argv = require('yargs').argv;
|
||||
|
||||
const file = 'https://speed.hetzner.de/100MB.bin';
|
||||
@@ -52,7 +52,7 @@ function setProgress(label, completedBytes, totalBytes, hash) {
|
||||
}
|
||||
|
||||
async function buffered(label) {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
const hash = crypto.createHash('blake2b512');
|
||||
|
||||
const imageRes = await bhttp.get(file, {
|
||||
onDownloadProgress(completedBytes, totalBytes) {
|
||||
@@ -67,7 +67,7 @@ async function buffered(label) {
|
||||
}
|
||||
|
||||
async function streamed(label) {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
const hash = crypto.createHash('blake2b512');
|
||||
hash.setEncoding('hex');
|
||||
|
||||
const hashStream = new PassThrough();
|
||||
|
||||
+2
-2
@@ -5,13 +5,13 @@ const { PassThrough } = require('stream');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const bhttp = require('bhttp');
|
||||
const blake2 = require('blake2');
|
||||
const crypto = require('crypto');
|
||||
const sharp = require('sharp');
|
||||
|
||||
const url = 'https://thumbs.julesjordan.com/trial/content//upload/dl03/julesjordan/oil_overload_16_scene2//photos/alina_lopez_jules_jordan_com_77.jpg';
|
||||
|
||||
async function init() {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
const hash = crypto.createHash('blake2b512');
|
||||
hash.setEncoding('hex');
|
||||
|
||||
const res = await bhttp.get(url, {
|
||||
|
||||
Reference in New Issue
Block a user