23 Commits
Author SHA1 Message Date
DebaucheryLibrarian 0d727e91c6 1.257.6 2026-11-09 00:29:00 +01:00
DebaucheryLibrarian 6509b2a25a Removed 1280px poster attempt from Nubiles. 2026-11-09 00:28:58 +01:00
DebaucheryLibrarian 171a8c9944 1.257.5 2026-11-08 23:53:02 +01:00
DebaucheryLibrarian 87946580f1 Fixed Bang breaking if scene is missing poster. 2026-11-08 23:52:59 +01:00
DebaucheryLibrarian 34d0f31347 1.257.4 2026-11-08 23:40:52 +01:00
DebaucheryLibrarian ecacecfd62 Fixed broken Bang properties. 2026-11-08 23:40:50 +01:00
DebaucheryLibrarian 1db0833794 1.257.3 2026-11-08 23:12:15 +01:00
DebaucheryLibrarian 06163e5ba2 Fixed Nubiles photo album query. 2026-11-08 23:12:12 +01:00
DebaucheryLibrarian da71b8c179 Removed obsolete pm2 ecosystem file. 2026-11-08 22:13:26 +01:00
DebaucheryLibrarian 55ddacbf0d 1.257.2 2026-11-08 22:01:09 +01:00
DebaucheryLibrarian 81bed09ced Updated README. 2026-11-08 22:01:05 +01:00
DebaucheryLibrarian 8c3f1b0c80 1.257.1 2026-11-08 21:44:25 +01:00
DebaucheryLibrarian 732c72c6dd Skip scene search update if no scenes were added or updated. 2026-11-08 21:44:20 +01:00
DebaucheryLibrarian 43189f401a 1.257.0 2026-11-08 20:56:48 +01:00
DebaucheryLibrarian 02676efd0c Updated unprint. 2026-11-08 17:33:52 +01:00
DebaucheryLibrarian 12c27722ee Removed unused scripts using bhttp and removed dependency. 2026-11-08 17:18:10 +01:00
DebaucheryLibrarian 27be031068 Retired http utility in favor of unprint. 2026-11-08 16:37:18 +01:00
DebaucheryLibrarian b4d07bbaa6 Updated dependencies not locked by require. 2026-11-08 02:26:04 +01:00
DebaucheryLibrarian 06ae62436a Bumped Node to 26 LTS, replaced deprecated dependencies. Switched Nubiles from browser to request interface, seems to fare better with security wall. 2026-11-08 00:38:01 +01:00
DebaucheryLibrarian 98121e6162 1.256.5 2026-10-22 00:23:23 +02:00
DebaucheryLibrarian e447a0b6da Fixed Team Skeet channel slug scraper not matching numeric digits, updated Mylf DOM URL. 2026-10-22 00:23:21 +02:00
DebaucheryLibrarian 530d45448a 1.256.4 2026-10-21 21:29:15 +02:00
DebaucheryLibrarian 6a0759693c Fixed double slash in Family Strokes network URL. 2026-10-21 21:29:13 +02:00
51 changed files with 4760 additions and 21349 deletions
+2 -1
View File
@@ -5,11 +5,12 @@
"node": true
},
"parserOptions": {
"parser": "@babel/eslint-parser",
"ecmaVersion": 2024,
"sourceType": "script"
},
"rules": {
"import/no-unresolved": ["error", {"ignore": ["^yargs/helpers$", "^object-merge-advanced$"]}],
"import/extensions": ["error", "ignorePackages", {"pattern": {"js": "never", "mjs": "never", "jsx": "never"}, "pathGroupOverrides": [{"pattern": "@noble/hashes/**", "action": "ignore"}]}],
"indent": ["error", "tab"],
"no-tabs": "off",
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
+1 -1
View File
@@ -1 +1 @@
21.2.0
26.13.0
+8 -7
View File
@@ -2,9 +2,9 @@
**NSFW** - The latest releases from your favorite porn studios in one place.
## Install & run
Use [nvm](https://github.com/creationix/nvm) to install NodeJS v16.8.0 or newer. Download and unpack or clone this repository, and run the following in the root directory:
Use [nvm](https://github.com/creationix/nvm) to install NodeJS v26.13.0, and make sure it's active (`nvm use`). Download and unpack or clone this repository, and run the following in the root directory:
`npm install`
`npm ci`
### Set up database
Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user.
@@ -18,6 +18,11 @@ Do not modify `config/default.js`, but instead create a copy at `config/local.js
You can also use `npm run flush` to run both steps at once, and wipe the database completely later.
### Install browser
Unless configured to use unprint's remote browser server, a browser needs to be installed locally:
`npx patchright install chromium`
#### Networks and channels
To scrape the networks and channels available in the database, you can configure `include` and `exclude` lists. To include all available channels and only use the `exclude` list, leave the `include` parameter unconfigured. The `exclude` lists will exclude channels and child networks from networks on the `include` lists, but not vice versa. That is, if the `include` list includes a network and the `exclude` list excludes one of that network's channels, the channel will not be scraped. However, if the `include` list includes a channel, and the `exclude` list includes its parent network, the channel will be scraped.
@@ -50,11 +55,7 @@ exclude: {
},
```
### Building
To build traxxx, run the following command:
`npm run build`
### Thumbnails
To generate thumbnails for new logos and tag photos, install ImageMagick and run:
`npm run logos-thumbs`
-138
View File
@@ -1,138 +0,0 @@
const config = require('config');
const manticore = require('manticoresearch');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
const utilsApi = new manticore.UtilsApi(mantiClient);
const scenesFields = `
id int,
title text,
title_filtered text,
shoot_id text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
studio_id int,
studio_name text,
studio_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
movie_ids multi,
movies text,
serie_ids multi,
series text,
meta text,
date timestamp,
is_showcased bool,
created_at timestamp,
effective_date timestamp,
stashed int,
dupe_index int
`;
const moviesFields = `
id int,
title text,
title_filtered text,
channel_id int,
channel_name text,
channel_slug text,
network_id int,
network_name text,
network_slug text,
entity_ids multi,
actor_ids multi,
actors text,
tag_ids multi,
tags text,
meta text,
date timestamp,
has_cover bool,
created_at timestamp,
effective_date timestamp,
stashed int,
stashed_scenes int,
stashed_total int,
dupe_index int
`;
const actorsFields = `
id int,
name text,
slug string,
gender string,
date_of_birth timestamp,
country string,
has_avatar bool,
mass int,
height int,
cup string,
natural_boobs int,
penis_length int,
penis_girth int,
stashed int,
scenes int
`;
exports.up = async () => {
try {
await utilsApi.sql(`create table if not exists scenes (${scenesFields})`);
await utilsApi.sql(`create table if not exists scenes_stashed (
scene_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql(`create table if not exists movies (${moviesFields})`);
await utilsApi.sql(`create table if not exists movies_stashed (
movie_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`);
await utilsApi.sql(`create table if not exists actors_stashed (
actor_id int,
stash_id int,
user_id int,
created_at timestamp
)`);
/*
await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id'));
await knex.schema.alterTable('stashes_movies', (table) => table.increments('id'));
await knex.schema.alterTable('stashes_actors', (table) => table.increments('id'));
await knex.schema.alterTable('stashes_series', (table) => table.increments('id'));
*/
} catch (error) {
console.log(error);
}
};
exports.down = async (knex) => {
await utilsApi.sql('drop table if exists scenes');
await utilsApi.sql('drop table if exists scenes_stashed');
await utilsApi.sql('drop table if exists movies');
await utilsApi.sql('drop table if exists movies_stashed');
await utilsApi.sql('drop table if exists actors');
await utilsApi.sql('drop table if exists actors_stashed');
await knex.schema.alterTable('stashes_scenes', (table) => table.dropColumn('id'));
await knex.schema.alterTable('stashes_movies', (table) => table.dropColumn('id'));
await knex.schema.alterTable('stashes_actors', (table) => table.dropColumn('id'));
await knex.schema.alterTable('stashes_series', (table) => table.dropColumn('id'));
};
-14
View File
@@ -216,12 +216,7 @@ module.exports = {
},
bypass: {
browser: {
enable: false,
clientRetirement: 20,
hostnames: [ // these can run in the same browser session
'www.kink.com',
'store2.psmcdn.net', // Team Skeet API
],
remote: {
enable: false,
use: false,
@@ -229,15 +224,6 @@ module.exports = {
key: null,
},
},
cloudflare: {
enable: false,
auto: true, // try bypass when CF challenge is detected
path: 'http://localhost:8191/v1',
sharedHostnames: [ // these can run in the same browser session
'store2.psmcdn.net', // Team Skeet API
],
independentHostnames: [], // these must run in their own browser session
},
},
limits: {
default: {
-28
View File
@@ -1,28 +0,0 @@
const config = require('config');
module.exports = {
apps: [
{
name: 'traxxx',
script: 'src/init.js',
exec_mode: 'cluster',
instances: 2,
restart_delay: 3000,
args: '--server',
merge_logs: true,
env: {
NODE_ENV: 'production',
},
},
...(config.analytics.enabled ? [{
name: 'umami',
script: 'npm start',
cwd: process.env.UMAMI_DIR || '../umami',
restart_delay: 3000,
merge_logs: true,
env: {
NODE_ENV: 'production',
},
}] : []),
],
};
+4570 -18300
View File
File diff suppressed because it is too large Load Diff
+30 -108
View File
@@ -1,16 +1,13 @@
{
"name": "traxxx",
"version": "1.256.3",
"version": "1.257.6",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"engines": {
"node": ">=26"
},
"scripts": {
"start": "node -r source-map-support/register src/init.js",
"webpack": "webpack --env=production --mode=production",
"webpack-dev": "webpack --env=development --mode=development",
"webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
"babel": "babel src --source-maps -d dist",
"babel-watch": "babel src -w --source-maps -d dist",
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
"eslint": "eslint src/",
"eslint-watch": "esw --watch src/",
"knex": "knex",
@@ -38,135 +35,60 @@
"author": "DebaucheryLibrarian",
"license": "ISC",
"devDependencies": {
"@babel/cli": "^7.23.4",
"@babel/core": "^7.23.3",
"@babel/eslint-parser": "^7.23.3",
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
"@babel/preset-env": "^7.23.3",
"@babel/register": "^7.22.15",
"autoprefixer": "^10.4.16",
"babel-loader": "^9.1.3",
"babel-preset-airbnb": "^5.0.0",
"css-loader": "^6.8.1",
"eslint": "^8.54.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-plugin-import": "^2.29.0",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-react": "^7.33.2",
"eslint-plugin-vue": "^9.18.1",
"eslint-watch": "^8.0.0",
"eslint-webpack-plugin": "^4.0.1",
"mini-css-extract-plugin": "^2.7.6",
"node-sass": "^9.0.0",
"postcss-loader": "^7.3.3",
"raw-loader": "^4.0.2",
"sass-loader": "^13.3.2",
"style-loader": "^3.3.3",
"vue-loader": "^17.3.1",
"webpack": "^5.89.0",
"webpack-cli": "^5.1.4"
"eslint-watch": "^8.0.0"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.458.0",
"@aws-sdk/credential-provider-ini": "^3.458.0",
"@aws-sdk/lib-storage": "^3.458.0",
"@casl/ability": "^6.5.0",
"@faker-js/faker": "^8.3.1",
"@graphile-contrib/pg-order-by-related": "^1.0.0",
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
"@graphile/pg-aggregates": "^0.1.1",
"@faker-js/faker": "^10.6.0",
"@noble/hashes": "^2.4.0",
"@rsc-parser/react-client": "^1.1.2",
"@tensorflow/tfjs-core": "4.22.0",
"@tensorflow/tfjs-node": "^4.22.0",
"@vladmandic/human": "^3.3.6",
"acorn": "^8.11.2",
"array-equal": "^1.0.2",
"babel-polyfill": "^6.26.0",
"bhttp": "^1.2.8",
"blake2": "^5.0.0",
"bluebird": "^3.7.2",
"body-parser": "^1.20.2",
"bottleneck": "^2.19.5",
"casual": "^1.6.2",
"cheerio": "^1.0.0-rc.12",
"cli-confirm": "^1.0.1",
"clipboardy": "^2.3.0",
"cloudscraper": "^4.6.0",
"config": "^3.3.9",
"connect-session-knex": "^4.0.0",
"convert": "^4.14.0",
"cookie": "^0.6.0",
"css": "^3.0.0",
"csv-parse": "^6.1.0",
"config": "^5.0.1",
"convert": "^7.0.2",
"csv-stringify": "^6.4.4",
"date-fns": "^2.30.0",
"dayjs": "^1.11.10",
"date-fns": "^4.4.0",
"dompurify": "^3.0.6",
"ejs": "^3.1.9",
"escape-string-regexp": "^4.0.0",
"express": "^4.18.2",
"express-promise-router": "^4.1.1",
"express-query-boolean": "^2.0.0",
"express-react-views": "^0.11.0",
"express-session": "^1.17.3",
"face-api.js": "^0.22.2",
"file-type": "^18.7.0",
"escape-string-regexp": "^5.0.0",
"fluent-ffmpeg": "^2.1.3",
"fs-extra": "^11.1.1",
"graphile-build": "^4.14.0",
"graphile-utils": "^4.14.0",
"graphql": "^15.8.0",
"html-entities": "^2.4.0",
"https-proxy-agent": "^7.0.5",
"iconv-lite": "^0.6.3",
"inquirer": "^8.2.6",
"inquirer": "^14.2.5",
"inspector-api": "^1.4.8",
"jsdom": "^23.0.1",
"jsdom": "^31.0.0",
"knex": "^3.0.1",
"knex-migrate": "^1.7.4",
"longjohn": "^0.2.12",
"manticoresearch": "^4.0.0",
"merge-anything": "^5.1.7",
"mime": "^3.0.0",
"mitt": "^3.0.1",
"mime": "^4.1.2",
"moment": "^2.29.4",
"mysql": "^2.18.1",
"nanoid": "^3.3.7",
"nanoid": "^6.0.2",
"node-fetch": "^3.3.2",
"npm": "^11.7.0",
"object-merge-advanced": "^12.1.0",
"object-merge-advanced": "^14.2.5",
"object.omit": "^3.0.0",
"pg": "^8.11.3",
"playwright": "^1.57.0",
"playwright-core": "^1.57.0",
"postgraphile": "^4.14.0",
"postgraphile-plugin-connection-filter": "^2.3.0",
"promise-task-queue": "^1.2.0",
"prop-types": "^15.8.1",
"puppeteer": "^24.4.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"redis": "^4.6.11",
"sharp": "^0.33.0-rc.2",
"showdown": "^2.1.0",
"redis": "^6.3.1",
"sharp": "^0.35.5",
"source-map-support": "^0.5.21",
"template-format": "^1.2.5",
"tippy.js": "^6.3.7",
"tough-cookie": "^4.1.3",
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^7.24.7",
"unprint": "^0.19.20",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",
"videojs-vr": "^2.0.0",
"vue": "^3.3.9",
"vue-router": "^4.2.5",
"vuex": "^4.1.0",
"why-is-node-running": "^2.2.2",
"template-format": "^2.0.0",
"undici": "^8.12.4",
"unprint": "^1.0.0-beta.2",
"why-is-node-running": "^3.2.3",
"winston": "^3.11.0",
"winston-daily-rotate-file": "^4.7.1",
"yargs": "^17.7.2"
"winston-daily-rotate-file": "^5.0.0",
"yargs": "^18.2.0"
},
"allowScripts": {
"sharp": true,
"@tensorflow/tfjs-node": true,
"core-js": false
}
}
+1 -1
View File
@@ -172,7 +172,7 @@ const networks = [
{
slug: 'familystrokes',
name: 'Family Strokes',
url: 'https://www.familystrokes.com/',
url: 'https://www.familystrokes.com',
parent: 'paperstreetmedia',
parameters: {
endpoint: 'familybundle',
+6 -24
View File
@@ -6027,56 +6027,41 @@ const sites = [
slug: 'sexuallybroken',
name: 'Sexually Broken',
alias: ['seb'],
url: 'https://www.sexuallybroken.com',
url: 'https://www.insex.com/is/home.php?d=sexuallybroken.com',
tags: ['bdsm'],
parent: 'insex',
parameters: {
latest: 'https://www.sexuallybroken.com/sb',
},
},
{
slug: 'infernalrestraints',
name: 'Infernal Restraints',
alias: ['infr'],
url: 'https://www.infernalrestraints.com',
url: 'https://www.insex.com/is/home.php?d=infernalrestraints.com',
tags: ['bdsm'],
parent: 'insex',
parameters: {
latest: 'https://www.infernalrestraints.com/ir',
},
},
{
slug: 'hardtied',
name: 'Hardtied',
alias: ['ht'],
url: 'https://www.hardtied.com',
url: 'https://www.insex.com/is/home.php?d=hardtied.com',
tags: ['bdsm'],
parent: 'insex',
parameters: {
latest: 'https://www.hardtied.com/ht',
},
},
{
slug: 'realtimebondage',
name: 'Real Time Bondage',
alias: ['rtb'],
url: 'https://www.realtimebondage.com',
url: 'https://www.insex.com/is/home.php?d=realtimebondage.com',
tags: ['bdsm', 'live'],
parent: 'insex',
parameters: {
latest: 'https://www.realtimebondage.com/rtb',
},
},
{
slug: 'topgrl',
name: 'TopGrl',
alias: ['tg'],
url: 'https://www.topgrl.com',
url: 'https://www.insex.com/is/home.php?d=topgrl.com',
tags: ['bdsm', 'femdom'],
parent: 'insex',
parameters: {
latest: 'https://www.topgrl.com/tg',
},
},
{
slug: 'paintoy',
@@ -8578,12 +8563,9 @@ const sites = [
{
slug: 'mylfdom',
name: 'MYLF Dom',
url: 'https://www.mylfdom.com',
url: 'https://www.mylf.com/series/mylfdom-ts',
tags: ['bdsm', 'milf'],
parent: 'mylf',
parameters: {
endpoint: 'mylf_dom',
},
},
// NAUGHTY AMERICA
{
+1 -1
View File
@@ -1,7 +1,7 @@
const Promise = require('bluebird');
const casual = require('casual');
const fs = require('fs-extra');
const nanoid = require('nanoid');
const { nanoid } = require('nanoid');
const stringify = Promise.promisify(require('csv-stringify'));
const path = require('path');
+3 -7
View File
@@ -4,11 +4,11 @@ const config = require('config');
const util = require('util');
const Promise = require('bluebird');
const moment = require('moment');
const blake2 = require('blake2');
const { blake2b } = require('@noble/hashes/blake2.js');
const DOMPurify = require('dompurify');
const { JSDOM } = require('jsdom');
const omit = require('object.omit');
const inquirer = require('inquirer');
const inquirer = require('inquirer').default;
const unprint = require('unprint');
const { window } = new JSDOM('');
@@ -413,11 +413,7 @@ async function curateProfile(profile, actor) {
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
const hasher = curatedProfile.description && blake2
.createHash('blake2b', { digestLength: 24 })
.update(Buffer.from(slugify(curatedProfile.description)));
curatedProfile.descriptionHash = curatedProfile.description && hasher.digest('hex');
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(Buffer.from(slugify(curatedProfile.description)), { dkLen: 24 })).toString('hex');
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
+12 -8
View File
@@ -5,13 +5,12 @@ const startTime = new Date();
const config = require('config');
const util = require('util');
const unprint = require('unprint');
const log = require('why-is-node-running');
const log = require('why-is-node-running').default;
const Inspector = require('inspector-api');
const fs = require('fs').promises;
const { format, intervalToDuration } = require('date-fns');
const argv = require('./argv');
const http = require('./utils/http');
const logger = require('./logger')(__filename);
const knex = require('./knex');
@@ -34,7 +33,7 @@ const unprintOptions = {
logErrors: false,
timeout: argv.requestTimeout,
userAgent: 'traxxx',
browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
// browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
apiUserAgent: 'traxxx',
clientRetirement: config.bypass.browser.clientRetirement,
remote: config.bypass.browser.remote,
@@ -210,7 +209,7 @@ async function init() {
}
if (argv.request) {
const res = await http[argv.requestMethod](argv.request);
const res = await unprint.request(argv.request, { method: argv.requestMethod, interface: 'request' });
console.log(res.status, res.body);
}
@@ -238,7 +237,7 @@ async function init() {
const movieUrls = (argv.movie || []).concat(moviesFromFile || []);
const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;
const deepMovies = movieUrls.length > 0 || sceneMovies.length > 0 ? await fetchMovies([...movieUrls, ...sceneMovies]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
@@ -256,9 +255,15 @@ async function init() {
await associateMovieScenes(storedMovies, allScenes);
// refreshing the materialized views is slow, skip when nothing was stored (e.g. --request)
if (allScenes.length > 0) {
await updateSceneSearch(allScenes.map((scene) => scene.id));
}
if (storedMovies.length > 0) {
await updateMovieSearch(storedMovies.map((movie) => movie.id));
}
}
logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() }))
.filter(([, value]) => value > 0)
@@ -269,12 +274,11 @@ async function init() {
logger.error(error);
}
await http.destroyBypassSessions();
await http.destroyBrowserSessions();
await unprint.closeAllBrowsers();
await unprint.closeAgents();
knex.destroy();
redis.disconnect();
redis.destroy();
done = true;
}
+2 -1
View File
@@ -2,6 +2,7 @@
const config = require('config');
const yargs = require('yargs');
const { hideBin } = require('yargs/helpers');
const moment = require('moment');
function interpretDate(after, ignoreIfEmpty = false) {
@@ -27,7 +28,7 @@ function interpretDate(after, ignoreIfEmpty = false) {
.toDate();
}
const { argv } = yargs
const { argv } = yargs(hideBin(process.argv))
.command('npm start')
.command('update', 'fetch latest updates', () => {
console.log('UPDATING!');
-110
View File
@@ -1,110 +0,0 @@
'use strict';
const config = require('config');
const util = require('util');
const crypto = require('crypto');
const knex = require('./knex');
const { curateUser, fetchUser } = require('./users');
const { HttpError } = require('./errors');
const scrypt = util.promisify(crypto.scrypt);
async function verifyPassword(password, storedPassword) {
const [salt, hash] = storedPassword.split('/');
const hashedPassword = (await scrypt(password, salt, 64)).toString('hex');
if (hashedPassword === hash) {
return true;
}
throw new HttpError('Username or password incorrect', 401);
}
async function login(credentials) {
if (!config.auth.login) {
throw new HttpError('Authentication is disabled', 405);
}
const user = await fetchUser(credentials.username.trim(), true);
if (!user) {
throw new HttpError('Username or password incorrect', 401);
}
await verifyPassword(credentials.password, user.password);
await knex('users')
.update('last_login', 'NOW()')
.where('id', user.id);
return curateUser(user);
}
async function signup(credentials) {
if (!config.auth.signup) {
throw new HttpError('Authentication is disabled', 405);
}
const curatedUsername = credentials.username.trim();
if (!curatedUsername) {
throw new HttpError('Username required', 400);
}
if (curatedUsername.length < config.auth.usernameLength[0]) {
throw new HttpError('Username is too short', 400);
}
if (curatedUsername.length > config.auth.usernameLength[1]) {
throw new HttpError('Username is too long', 400);
}
if (!config.auth.usernamePattern.test(curatedUsername)) {
throw new HttpError('Username contains invalid characters', 400);
}
if (!credentials.email) {
throw new HttpError('E-mail required', 400);
}
if (credentials.password?.length < 3) {
throw new HttpError('Password must be 3 characters or longer', 400);
}
const existingUser = await knex('users')
.where('username', curatedUsername)
.orWhere('email', credentials.email)
.first();
if (existingUser) {
throw new HttpError('Username or e-mail already in use', 409);
}
const salt = crypto.randomBytes(16).toString('hex');
const hashedPassword = (await scrypt(credentials.password, salt, 64)).toString('hex');
const storedPassword = `${salt}/${hashedPassword}`;
const [{ id: userId }] = await knex('users')
.insert({
username: curatedUsername,
email: credentials.email,
password: storedPassword,
})
.returning('id');
await knex('stashes').insert({
user_id: userId,
name: 'Favorites',
slug: 'favorites',
public: false,
primary: true,
});
return fetchUser(userId);
}
module.exports = {
login,
signup,
};
+7 -1
View File
@@ -3,7 +3,13 @@
const config = require('config');
const path = require('path');
const fsPromises = require('fs').promises;
const Human = require('@vladmandic/human').default;
const util = require('util');
// tfjs-node still uses util helpers that were removed (isNullOrUndefined) or deprecated with a runtime warning (isArray) in Node 23+, must be set before it loads
util.isNullOrUndefined ??= (value) => value === null || value === undefined;
util.isArray = Array.isArray;
const Human = require('@vladmandic/human').default; // eslint-disable-line import/order
const knex = require('./knex');
const logger = require('./logger')(__filename);
-10
View File
@@ -10,7 +10,6 @@ const include = require('./utils/argv-include')(argv);
const { fetchReleaseEntities, urlToHostname } = require('./entities');
const logger = require('./logger')(__filename);
const getRecursiveParameters = require('./utils/get-recursive-parameters');
const windows = require('./utils/http-windows');
const waitImmediate = util.promisify(setImmediate);
@@ -142,19 +141,10 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options)
: await fetchMovie(layoutScraper, baseRelease.url, entity, baseRelease, options);
const pathname = baseRelease.path || (baseRelease.url && new URL(baseRelease.url).pathname.replace(/\//g, '_'));
if (rawScrapedRelease) {
delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
}
if (windows.has(pathname)) {
logger.debug(`Closing window for ${pathname}`);
windows.get(pathname).close();
windows.delete(pathname);
}
await waitImmediate;
// logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
+1 -1
View File
@@ -1,7 +1,7 @@
'use strict';
const config = require('config');
const inquirer = require('inquirer');
const inquirer = require('inquirer').default;
const logger = require('./logger')(__filename);
const argv = require('./argv');
-1
View File
@@ -1,4 +1,3 @@
require('babel-polyfill');
const init = require('./app');
init();
-12
View File
@@ -1,12 +0,0 @@
const config = require('config');
const manticore = require('manticoresearch');
const mantiClient = new manticore.ApiClient();
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
module.exports = {
searchApi: new manticore.SearchApi(mantiClient),
indexApi: new manticore.IndexApi(mantiClient),
utilsApi: new manticore.UtilsApi(),
};
+23 -12
View File
@@ -8,11 +8,11 @@ const fsPromises = require('fs').promises;
const path = require('path');
const stream = require('stream');
const { nanoid } = require('nanoid/non-secure');
const mime = require('mime');
const mime = require('mime').default;
// const fileType = require('file-type');
const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp');
const blake2 = require('blake2');
const { blake2b } = require('@noble/hashes/blake2.js');
const { format } = require('date-fns');
const taskQueue = require('promise-task-queue');
const unprint = require('unprint');
@@ -23,13 +23,15 @@ const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
const logger = require('./logger')(__filename);
const argv = require('./argv');
const knex = require('./knex');
const http = require('./utils/http');
const batchInsert = require('./utils/batch-insert');
const chunk = require('./utils/chunk');
const { fetchEntityReleaseIds } = require('./entity-releases');
const streamQueue = taskQueue();
// carried over from the legacy HTTP module, unprint's raw request interface would identify as traxxx
const mediaUserAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36';
const s3 = new S3Client({
region: 'eu-central-1',
endpoint: 'https://s3.eu-central-1.wasabisys.com',
@@ -637,24 +639,28 @@ async function storeFile(media, options) {
}
async function fetchHttpSource(source, tempFileTarget, hashStream) {
const res = await http.get(source.src, {
limits: 'media',
const res = await unprint.get(source.src, {
interface: 'request',
limiter: 'media',
headers: {
'user-agent': mediaUserAgent,
// explicit host not allowed in HTTP/2
// host: new URL(source.src).hostname,
// ...(source.host && { host: source.host }),
...(source.referer && { referer: source.referer }),
},
stream: true, // sources are fetched in parallel, don't gobble up memory
followRedirects: source.followRedirects,
tls: { rejectUnauthorized: false },
// sources are fetched in parallel, stream to file instead of gobbling up memory
transforms: [hashStream],
destination: tempFileTarget,
...(source.interval && { interval: source.interval }),
...(source.concurrency && { concurrency: source.concurrency }),
// undefined would override the defaults
...(typeof source.followRedirects === 'boolean' && { followRedirects: source.followRedirects }),
...((argv.mediaInterval ?? source.interval) && { interval: argv.mediaInterval ?? source.interval }),
...((argv.mediaConcurrency ?? source.concurrency) && { concurrency: argv.mediaConcurrency ?? source.concurrency }),
});
if (!res.ok) {
throw new Error(`Response ${res.status} not OK`);
throw new Error(`Response ${res.status} not OK${res.statusText ? ` (${res.statusText})` : ''}`); // status 200 if stream was interrupted
}
return {
@@ -723,7 +729,7 @@ async function fetchSource(source, baseMedia) {
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
let tempFileTarget;
const hasher = blake2.createHash('blake2b', { digestLength: 24 });
const hasher = blake2b.create({ dkLen: 24 });
let size = 0;
@@ -743,7 +749,7 @@ async function fetchSource(source, baseMedia) {
? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
: await fetchHttpSource(source, tempFileTarget, hashStream);
const hash = hasher.digest('hex');
const hash = Buffer.from(hasher.digest()).toString('hex');
const [type, subtype] = mimetype.split('/');
const extension = mime.getExtension(mimetype);
@@ -772,6 +778,8 @@ async function fetchSource(source, baseMedia) {
hashStream.destroy();
tempFileTarget?.destroy();
await fsPromises.unlink(tempFilePath).catch(() => { /* probably never created */ });
if (error.code !== 'VERIFY_TYPE') {
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
@@ -914,6 +922,9 @@ async function storeMedias(baseMedias, options) {
async (baseMedia) => storeFile(baseMedia, options),
{ concurrency: 100 }, // don't overload disk
);
} else {
// duplicates aren't stored, clean up their fetched temp files
await Promise.all(existingHashMedias.map(async (media) => media.file?.path && fsPromises.unlink(media.file.path).catch(() => { /* probably already removed */ })));
}
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
+1 -1
View File
@@ -1,6 +1,6 @@
'use strict';
const inquirer = require('inquirer');
const inquirer = require('inquirer').default;
const logger = require('./logger')(__filename);
const knex = require('./knex');
+12 -13
View File
@@ -62,7 +62,7 @@ function scrapeAll(scenes, entity) {
release.title = query.content('a > span.block');
release.date = query.date('a + div', 'MMM DD, YYYY');
release.duration = query.duration('[data-videopreview-target="duration"]', { attribute: 'data-content' });
release.duration = query.duration('[data-videopreview-target="duration"], .video_duration', { attribute: 'data-content' });
release.actors = query.elements('a + div a[href*="/pornstar"]').map((el) => ({
name: unprint.query.content(el),
@@ -70,9 +70,11 @@ function scrapeAll(scenes, entity) {
}));
const poster = query.img('img[data-videopreview-target="image"]');
const posterUrl = new URL(poster);
if (poster) {
try {
const posterUrl = new URL(poster);
release.poster = [
`${posterUrl.origin}${posterUrl.pathname}`,
posterUrl.href,
@@ -82,18 +84,15 @@ function scrapeAll(scenes, entity) {
'application/octet-stream': 'image/jpeg',
},
}));
} catch (error) {
// nothing needed
}
}
const videoData = query.json('a', { attribute: 'data-videopreview-sources-value' });
if (videoData) {
release.teaser = [
videoData.mp4_large,
videoData.webm_large,
videoData.mp4,
videoData.webm,
];
}
query.url('a[data-videopreview-mp4-large-value]', { attribute: 'data-videopreview-mp4-large-value' }),
query.url('a[data-videopreview-mp4-value]', { attribute: 'data-videopreview-mp4-value' }),
].filter(Boolean);
return release;
});
@@ -150,7 +149,7 @@ async function scrapeScene({ query }, { url, entity }) {
}));
}
release.tags = query.contents('div[data-controller] .genres');
release.tags = query.contents('a.genres');
const sourcesData = query.json('.video-container [data-videopreview-sources-value]', { attribute: 'data-videopreview-sources-value' });
@@ -175,7 +174,7 @@ async function scrapeScene({ query }, { url, entity }) {
|| query.attribute('meta[property="og:video"]')
|| query.video('video[data-videocontainer-target] source');
release.photos = query.sourceSets('div[data-controller] a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
release.photos = query.sourceSets('a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
src,
expectType: {
'application/octet-stream': 'image/jpeg',
+18 -94
View File
@@ -21,51 +21,33 @@ function scrapeLatest(scenes, site) {
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
console.log(release);
return release;
});
}
function scrapeLatestLegacy(scenes, site) {
return scenes.map(({ query }) => {
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
const release = {};
async function fetchLatest(site, page = 1) {
const domain = new URL(site.url).searchParams.get('d');
const titleEl = query.element('.galleryTitleText, .articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
if (date) {
release.title = title.trim();
release.date = date;
} else {
// title should contain date instead, not applicable in brief mode
release.title = title.slice(title.indexOf(':') + 1).trim();
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
if (!domain) {
return null;
}
release.actors = actors.map((actor) => actor.trim());
const url = `${site.origin}/is/home.php?o=newest&d=${domain}&p=${page}`;
const description = query.content('.articleCopyText');
if (description) release.description = description.slice(0, description.lastIndexOf('('));
const duration = query.duration('.articleCopyText a:nth-child(2)');
if (duration) release.duration = duration;
release.likes = query.number('.articlePostDateText td:nth-child(3)');
const cover = query.img('a img', { origin: site.url });
release.covers = [[
cover.replace('_thumbnail', ''),
cover,
]];
return release;
const res = await unprint.get(url, {
selectAll: '.column.is-3-desktop',
cookies: { consent: 'yes' },
});
// (UNSEXUALLY) BROKEN, abandoned sites, low priority
if (res.ok) {
return scrapeLatest(res.context, site);
}
return res.status;
}
async function scrapeScene({ query }, channel, include, cookies) {
@@ -97,60 +79,6 @@ async function scrapeScene({ query }, channel, include, cookies) {
return release;
}
function scrapeSceneLegacy({ query }, { entity: site }) {
const release = {};
const titleEl = query.element('.articleTitleText');
const [title, ...actors] = titleEl.textContent.split('|');
const url = unprint.query.attribute(titleEl, 'a', 'href');
[release.entryId] = url.split('/').slice(-2);
release.url = `${site.url}${url}`;
release.title = title.trim();
release.description = query.content('.articleCopyText');
release.actors = actors.map((actor) => actor.trim());
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
const [cover, ...photos] = query.imgs('img[src*="images"]');
release.covers = [cover];
release.photos = photos;
release.poster = query.poster();
const trailer = query.video('source');
if (trailer) release.trailer = { src: trailer };
return release;
}
async function fetchLatest(site, page = 1) {
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
const res = await unprint.get(url, {
selectAll: 'body > .columns .column',
cookies: { consent: 'yes' },
});
if (res.ok) {
return scrapeLatest(res.context, site);
}
return res.status;
}
async function fetchLatestLegacy(site, page = 1) {
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
const res = await unprint.get(url); // JSON containing html as a property
if (res.ok && res.data?.html) {
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
}
return res.status;
}
async function fetchScene(url, site, baseRelease, include) {
const res = await unprint.get(url, {
cookies: { consent: 'yes' },
@@ -166,8 +94,4 @@ async function fetchScene(url, site, baseRelease, include) {
module.exports = {
fetchLatest,
fetchScene,
legacy: {
fetchLatest: fetchLatestLegacy,
scrapeScene: scrapeSceneLegacy,
},
};
+18 -13
View File
@@ -22,12 +22,13 @@ function stripQuery(link) {
async function getPhotos(albumUrl) {
// aggressive Turnstile protection calls for browser
const res = await unprint.browser(albumUrl, {
const res = await unprint.get(albumUrl, {
selectAll: '.photo-thumb',
interface: 'request',
});
if (res.ok) {
return res.context.map(({ query }) => query.sourceSet('source, picture img')).filter(Boolean);
return res.context.map(({ query }) => query.img('picture img')).filter(Boolean);
}
return [];
@@ -53,14 +54,10 @@ function scrapeAll(scenes, entity) {
release.actors = query.content('.models', { trim: false })?.trim().split(/\s{2,}/);
}
const poster = query.sourceSet('img', 'data-srcset')?.[0]
// old 640px to 1280px replace won't work anymore with token in URL
release.poster = query.sourceSet('img', 'data-srcset')?.[0]
|| query.sourceSet('img', 'data-src')?.[0];
release.poster = poster && [
poster.replace('_640', '_1280'),
poster,
];
release.stars = query.number('.rating');
release.likes = query.number('.likes');
@@ -73,7 +70,8 @@ function scrapeAll(scenes, entity) {
async function fetchLatest(site, page = 1) {
const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
const res = await unprint.browser(url, {
const res = await unprint.get(url, {
interface: 'request',
selectAll: '.content-grid-item',
});
@@ -88,7 +86,8 @@ async function fetchUpcoming(site) {
if (site.parameters?.upcoming) {
const url = `${site.url}/video/upcoming`;
const res = await unprint.browser(url, {
const res = await unprint.get(url, {
interface: 'request',
selectAll: '.content-grid-item',
});
@@ -140,7 +139,9 @@ async function scrapeScene({ query }, { url, entity, include }) {
}
async function fetchScene(url, entity, _baseRelease, include) {
const res = await unprint.browser(url);
const res = await unprint.get(url, {
interface: 'request',
});
if (res.ok) {
return scrapeScene(res.context, { url, entity, include });
@@ -184,7 +185,9 @@ async function findModel(actor, entity) {
const url = `${origin}/model/alpha/${firstLetter}`;
const resModels = await unprint.browser(url);
const resModels = await unprint.get(url, {
interface: 'request',
});
if (!resModels.ok) {
return resModels.status;
@@ -214,7 +217,9 @@ async function fetchProfile(actor, { entity }) {
const model = await findModel(actor, entity);
if (model) {
const resModel = await unprint.browser(model.url);
const resModel = await unprint.get(model.url, {
interface: 'request',
});
if (resModel.ok) {
return scrapeProfile(resModel.context, model.avatar);
+2 -6
View File
@@ -1,15 +1,11 @@
'use strict';
const blake2 = require('blake2');
const { blake2b } = require('@noble/hashes/blake2.js');
const unprint = require('unprint');
function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 });
hash.update(Buffer.from(identifier));
return hash.digest('hex');
return Buffer.from(blake2b(Buffer.from(identifier), { dkLen: 8 })).toString('hex');
}
function extractMaleModelsFromTags(tagContainer) {
+1 -1
View File
@@ -1,7 +1,7 @@
'use strict';
const unprint = require('unprint');
const mime = require('mime');
const mime = require('mime').default;
const slugify = require('../utils/slugify');
const { convert } = require('../utils/convert');
+1 -1
View File
@@ -1,7 +1,7 @@
'use strict';
const unprint = require('unprint');
const escapeRegexp = require('escape-string-regexp');
const escapeRegexp = require('escape-string-regexp').default;
const slugify = require('../utils/slugify');
const capitalize = require('../utils/capitalize');
+14 -29
View File
@@ -47,11 +47,7 @@ function scrapeAll(scenes, tilesByEntryId, channel) {
release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
const posterBackground = query.style('.player');
if (posterBackground?.background) {
release.poster = posterBackground.background.match(/url\((.*)\)/)?.[1]?.trim();
}
release.poster = query.background('.player');
release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
unprint.query.url(linkEl, null),
@@ -121,8 +117,7 @@ function scrapeAllTiles(tiles, channel) {
release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
const posterBackground = query.style('.scene-img');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.scene-img');
if (posterUrl) {
release.poster = [
@@ -199,8 +194,7 @@ function scrapeAllCospuri(scenes, channel) {
release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
const posterBackground = query.style('.scene-thumb');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.scene-thumb');
if (posterUrl) {
release.poster = [
@@ -260,8 +254,7 @@ function scrapeAllFellatio(scenes, channel) {
release.tags = query.contents('.tour-data a[href*="tag/"]');
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -302,8 +295,7 @@ function scrapeAllHandjob(scenes, _channel) {
release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -348,8 +340,7 @@ function scrapeAllLegs(scenes, channel) {
release.tags = query.contents('a[href*="tag/"]');
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -385,8 +376,7 @@ function scrapeAllFacefuck(scenes) {
release.description = query.content('.infotxt');
release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -425,8 +415,7 @@ function scrapeAllTrans(scenes) {
release.duration = query.duration('.sample-info');
release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = [
@@ -436,7 +425,7 @@ function scrapeAllTrans(scenes) {
];
}
release.photos = curatePhotos(albumContext?.query.styles('.sample-lg, .sample-thumb').map((style) => style['background-image']?.match(/url\((.*)\)/)?.[1]));
release.photos = curatePhotos(albumContext?.query.backgrounds('.sample-lg, .sample-thumb'));
release.trailer = query.video();
release.entryId = entryIdFromMedia(release);
@@ -486,8 +475,7 @@ function scrapeAllLesbianTiles(scenes, channel) {
release.tags = query.contents('.content-tags a');
const posterBackground = query.style('.vidthumb');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.vidthumb');
if (posterUrl) {
release.poster = [
@@ -522,8 +510,7 @@ function scrapeAllLesbian(scenes, channel, tiles) {
release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -650,8 +637,7 @@ function scrapeSceneBuffet({ query }, { url, entity }) {
release.tags = query.contents('.tag-list a');
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = [
@@ -696,8 +682,7 @@ function scrapeSceneCospuri({ query }, { url, entity }) {
release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
const posterBackground = query.style('.player');
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
const posterUrl = query.background('.player');
if (posterUrl) {
release.poster = posterUrl;
@@ -788,7 +773,7 @@ function scrapeProfile({ query }, _channel, url) {
const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
|| query.img('.costume-bg', { attribute: 'data-img' })
|| query.style('.model-profile, #profile, .carousel-item')?.['background-image']?.match(/url\((.*)\)/)?.[1];
|| query.background('.model-profile, #profile, .carousel-item');
if (avatar) {
profile.avatar = Array.from(new Set([
+3 -3
View File
@@ -47,7 +47,7 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
// release.entryId = scene.id; // legacy
release.entryId = scene.itemId;
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
release.title = scene.title;
release.description = scene.description;
@@ -58,7 +58,7 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
name: model.modelName || model.name || model.title,
avatar: model.img,
gender: model.gender,
url: `${channel.url}/models/${model.modelId || model.id}`,
url: `${channel.origin}/models/${model.modelId || model.id}`,
}));
if (scene.img) {
@@ -94,7 +94,7 @@ function scrapeAll(scenes, channel, parameters) {
async function fetchLatest(channel, page = 1, { parameters }) {
// url: 'https://www.pervz.com/series/pervz-features',
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
const url = seriesId
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
+2 -2
View File
@@ -241,8 +241,8 @@ function generateActors(release) {
: Math.floor(Math.random() * 3) + 2;
return Array.from({ length }, () => ({
name: faker.name
.findName()
name: faker.person
.fullName()
.split(' ')
.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
.join(' '),
-312
View File
@@ -1,312 +0,0 @@
'use strict';
const config = require('config');
const knex = require('./knex');
const { indexApi } = require('./manticore');
const { HttpError } = require('./errors');
const slugify = require('./utils/slugify');
const logger = require('./logger')(__filename);
let lastActorsViewRefresh = 0;
function curateStash(stash) {
if (!stash) {
return null;
}
const curatedStash = {
id: stash.id,
name: stash.name,
slug: stash.slug,
primary: stash.primary,
};
return curatedStash;
}
function curateStashEntry(stash, user) {
const curatedStashEntry = {
user_id: user.id,
name: stash.name,
slug: slugify(stash.name),
public: false,
};
return curatedStashEntry;
}
async function fetchStash(stashId, sessionUser) {
if (!sessionUser) {
throw new HttpError('You are not authenthicated', 401);
}
const stash = await knex('stashes')
.where({
id: stashId,
user_id: sessionUser.id,
})
.first();
if (!stash) {
throw new HttpError('You are not authorized to access this stash', 403);
}
return curateStash(stash);
}
async function fetchStashes(domain, itemId, sessionUser) {
const stashes = await knex(`stashes_${domain}s`)
.select('stashes.*')
.where({
[`${domain}_id`]: itemId,
user_id: sessionUser.id,
})
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
return stashes.map((stash) => curateStash(stash));
}
async function createStash(newStash, sessionUser) {
if (!sessionUser) {
throw new HttpError('You are not authenthicated', 401);
}
try {
const stash = await knex('stashes')
.insert(curateStashEntry(newStash, sessionUser))
.returning('*');
return curateStash(stash);
} catch (error) {
if (error.routine === '_bt_check_unique') {
throw new HttpError('Stash name should be unique', 409);
}
throw error;
}
}
async function updateStash(stashId, newStash, sessionUser) {
if (!sessionUser) {
throw new HttpError('You are not authenthicated', 401);
}
const stash = await knex('stashes')
.where({
id: stashId,
user_id: sessionUser.id,
})
.update(newStash)
.returning('*');
if (!stash) {
throw new HttpError('You are not authorized to modify this stash', 403);
}
return curateStash(stash);
}
async function removeStash(stashId, sessionUser) {
if (!sessionUser) {
throw new HttpError('You are not authenthicated', 401);
}
const removed = await knex('stashes')
.where({
id: stashId,
user_id: sessionUser.id,
primary: false,
})
.delete();
if (removed === 0) {
throw new HttpError('Unable to remove this stash', 400);
}
}
async function refreshActorsView() {
if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
// don't refresh actors view more than once an hour
lastActorsViewRefresh = new Date();
logger.debug('Refreshing actors view');
return knex.schema.refreshMaterializedView('actors_meta');
}
logger.silly('Skipping actors view refresh');
return false;
}
async function stashActor(actorId, stashId, sessionUser) {
const stash = await fetchStash(stashId, sessionUser);
const [stashed] = await knex('stashes_actors')
.insert({
stash_id: stash.id,
actor_id: actorId,
})
.returning(['id', 'created_at']);
await indexApi.replace({
index: 'actors_stashed',
id: stashed.id,
doc: {
actor_id: actorId,
user_id: sessionUser.id,
stash_id: stashId,
created_at: Math.round(stashed.created_at.getTime() / 1000),
},
});
refreshActorsView();
return fetchStashes('actor', actorId, sessionUser);
}
async function stashScene(sceneId, stashId, sessionUser) {
const stash = await fetchStash(stashId, sessionUser);
const [stashed] = await knex('stashes_scenes')
.insert({
stash_id: stash.id,
scene_id: sceneId,
})
.returning(['id', 'created_at']);
await indexApi.replace({
index: 'scenes_stashed',
id: stashed.id,
doc: {
// ...doc.replace.doc,
scene_id: sceneId,
user_id: sessionUser.id,
stash_id: stashId,
created_at: Math.round(stashed.created_at.getTime() / 1000),
},
});
return fetchStashes('scene', sceneId, sessionUser);
}
async function stashMovie(movieId, stashId, sessionUser) {
const stash = await fetchStash(stashId, sessionUser);
const [stashed] = await knex('stashes_movies')
.insert({
stash_id: stash.id,
movie_id: movieId,
})
.returning(['id', 'created_at']);
await indexApi.replace({
index: 'movies_stashed',
id: stashed.id,
doc: {
movie_id: movieId,
user_id: sessionUser.id,
stash_id: stashId,
created_at: Math.round(stashed.created_at.getTime() / 1000),
},
});
return fetchStashes('movie', movieId, sessionUser);
}
async function unstashActor(actorId, stashId, sessionUser) {
await knex
.from('stashes_actors AS deletable')
.where('deletable.actor_id', actorId)
.where('deletable.stash_id', stashId)
.whereExists(knex('stashes_actors') // verify user owns this stash, complimentary to row-level security
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
.where('stashes_actors.stash_id', knex.raw('deletable.stash_id'))
.where('stashes.user_id', sessionUser.id))
.delete();
await indexApi.callDelete({
index: 'actors_stashed',
query: {
bool: {
must: [
{ equals: { actor_id: actorId } },
{ equals: { stash_id: stashId } },
{ equals: { user_id: sessionUser.id } },
],
},
},
});
refreshActorsView();
return fetchStashes('actor', actorId, sessionUser);
}
async function unstashScene(sceneId, stashId, sessionUser) {
await knex
.from('stashes_scenes AS deletable')
.where('deletable.scene_id', sceneId)
.where('deletable.stash_id', stashId)
.whereExists(knex('stashes_scenes') // verify user owns this stash, complimentary to row-level security
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
.where('stashes_scenes.stash_id', knex.raw('deletable.stash_id'))
.where('stashes.user_id', sessionUser.id))
.delete();
await indexApi.callDelete({
index: 'scenes_stashed',
query: {
bool: {
must: [
{ equals: { scene_id: sceneId } },
{ equals: { stash_id: stashId } },
{ equals: { user_id: sessionUser.id } },
],
},
},
});
return fetchStashes('scene', sceneId, sessionUser);
}
async function unstashMovie(movieId, stashId, sessionUser) {
await knex
.from('stashes_movies AS deletable')
.where('deletable.movie_id', movieId)
.where('deletable.stash_id', stashId)
.whereExists(knex('stashes_movies') // verify user owns this stash, complimentary to row-level security
.leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id')
.where('stashes_movies.stash_id', knex.raw('deletable.stash_id'))
.where('stashes.user_id', sessionUser.id))
.delete();
await indexApi.callDelete({
index: 'movies_stashed',
query: {
bool: {
must: [
{ equals: { movie_id: movieId } },
{ equals: { stash_id: stashId } },
{ equals: { user_id: sessionUser.id } },
],
},
},
});
return fetchStashes('movie', movieId, sessionUser);
}
module.exports = {
createStash,
curateStash,
removeStash,
stashActor,
stashScene,
stashMovie,
unstashScene,
unstashActor,
unstashMovie,
updateStash,
};
+6 -1
View File
@@ -2,7 +2,7 @@
const config = require('config');
const Promise = require('bluebird');
const { decode } = require('html-entities');
const { decode: decodeEntities } = require('html-entities');
const unprint = require('unprint');
const argv = require('./argv');
@@ -28,6 +28,11 @@ const commonContext = {
unprint,
};
// newer html-entities throws on non-strings, pass e.g. numeric titles through like 2.4 did
function decode(text) {
return typeof text === 'string' ? decodeEntities(text) : text;
}
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
-84
View File
@@ -1,84 +0,0 @@
'use strict';
const fs = require('fs').promises;
const bhttp = require('bhttp');
const slugify = require('../utils/slugify');
const { studios: oldStudios } = require('../../seeds/03_studios');
/*
const slugMap = {
thay_ksada: 'thayksada2',
};
*/
async function init() {
const res = await bhttp.get('https://pornbox.com/studio/list');
if (!res.statusCode === 200) {
throw new Error(`Failed to retrieve studios (${res.statusCode}): ${res.body.toString()}`);
}
const newStudios = res.body.map((studio) => {
const slug = slugify(studio.name, '');
const oldStudio = oldStudios.find((oldStudioX) => oldStudioX.slug === slug);
const tags = oldStudio?.tags || [];
const newStudio = {
name: studio.name,
slug,
originalSlug: slugify(studio.alias, ''),
url: `https://www.analvids.com/studios/${studio.alias}`,
parent: 'analvids',
};
if (oldStudio?.alias) {
newStudio.alias = oldStudio.alias;
}
if (studio.description && studio.description.toLowerCase() !== studio.name.toLowerCase()) {
newStudio.description = studio.description;
}
if (studio.directions?.includes('gay')) {
tags.push('gay');
}
if (studio.directions?.includes('lesbian')) {
tags.push('lesbian');
}
if (studio.directions?.includes('trans')) {
tags.push('transsexual');
}
if (tags.length > 0) {
newStudio.tags = Array.from(new Set(tags));
}
return newStudio;
});
await fs.writeFile('./analvids-studios.json', JSON.stringify(newStudios, null, 4));
newStudios.reduce((acc, studio) => {
const existingStudio = acc.get(studio.slug);
if (existingStudio) {
if (!acc.has(studio.originalSlug)) {
console.log('REWRITE', studio.slug, studio.originalSlug);
studio.slug = studio.originalSlug; // eslint-disable-line
}
console.log('slug already exists!', existingStudio, studio, studio.originalSlug, studio.slug);
}
acc.set(studio.slug, studio);
return acc;
}, new Map());
// console.log(newStudios);
}
init();
-18
View File
@@ -1,18 +0,0 @@
'use strict';
const bhttp = require('bhttp');
async function init() {
const res = await bhttp.head('https://www.kink.com/consent', {
headers: {
Host: 'www.kink.com',
'User-Agent': 'HTTPie/2.6.0',
Accept: '*/*',
Connection: 'keep-alive',
},
});
console.log(res.body.toString(), res.headers, res.statusCode);
}
init();
-40
View File
@@ -1,40 +0,0 @@
'use strict';
const fetch = require('node-fetch');
const express = require('express');
async function init() {
const res = await fetch('https://www.realitykings.com/scenes?site=45', {
method: 'HEAD',
headers: {
'user-agent': 'HTTPie/2.6.0',
'accept-encoding': 'gzip, deflate, br',
accept: '*/*',
connection: 'keep-alive',
},
});
console.log(res.status, res.headers);
const app = express();
app.get('/', (appReq, appRes) => {
console.log(appReq.headers);
appRes.status(204).send();
});
app.listen(8000, () => {
console.log('Listening on port 8000');
fetch('http://127.0.0.1:8000', {
headers: {
'user-agent': 'HTTPie/2.6.0',
'accept-encoding': 'gzip, deflate, br',
accept: '*/*',
connection: 'keep-alive',
},
});
});
}
init();
-17
View File
@@ -1,17 +0,0 @@
'use strict';
const express = require('express');
const app = express();
app.get('/', (req, res) => {
console.log(req.httpVersion);
console.log(req.headers);
console.log(req.body);
res.status(204).send();
});
app.listen(5555, () => {
console.log('Server started');
});
+1 -1
View File
@@ -2,7 +2,7 @@
// const util = require('util');
const unprint = require('unprint');
const args = require('yargs').argv;
const args = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
const knex = require('../knex');
const bulkInsert = require('../utils/bulk-insert');
-56
View File
@@ -1,56 +0,0 @@
'use strict';
const knex = require('./knex');
const { curateStash } = require('./stashes');
function curateUser(user) {
if (!user) {
return null;
}
const ability = [...(user.role_abilities || []), ...(user.abilities || [])];
const curatedUser = {
id: user.id,
username: user.username,
email: user.email,
emailVerified: user.email_verified,
identityVerified: user.identity_verified,
ability,
createdAt: user.created_at,
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
};
return curatedUser;
}
async function fetchUser(userId, raw) {
const user = await knex('users')
.select(knex.raw('users.*, users_roles.abilities as role_abilities, COALESCE(json_agg(stashes ORDER BY stashes.created_at) FILTER (WHERE stashes.id IS NOT NULL), \'[]\') as stashes'))
.modify((builder) => {
if (typeof userId === 'number') {
builder.where('users.id', userId);
}
if (typeof userId === 'string') {
builder
.where('users.username', userId)
.orWhere('users.email', userId);
}
})
.leftJoin('users_roles', 'users_roles.role', 'users.role')
.leftJoin('stashes', 'stashes.user_id', 'users.id')
.groupBy('users.id', 'users_roles.role')
.first();
if (raw) {
return user;
}
return curateUser(user);
}
module.exports = {
curateUser,
fetchUser,
};
-118
View File
@@ -1,118 +0,0 @@
'use strict';
const bhttp = require('bhttp');
const Promise = require('bluebird');
const fsPromises = require('fs').promises;
const fs = require('fs');
const { PassThrough } = require('stream');
const blake2 = require('blake2');
const argv = require('yargs').argv;
const file = 'https://speed.hetzner.de/100MB.bin';
// const file = 'https://speed.hetzner.de/1GB.bin';
// const file = 'https://speed.hetzner.de/10GB.bin';
function getMemoryUsage() {
return process.memoryUsage().rss / (10 ** 6);
}
const stats = {
peakMemoryUsage: getMemoryUsage(),
done: false,
downloads: {},
};
function render() {
const downloads = Object.entries(stats.downloads);
process.stdout.clearScreenDown();
process.stdout.write(`peak memory: ${stats.peakMemoryUsage.toFixed(2)} MB\n`);
downloads.forEach(([download, progress]) => {
process.stdout.write(`${download}: ${progress}${typeof progress === 'string' ? '' : '%'}\n`);
});
process.stdout.moveCursor(0, -(downloads.length + 1));
process.stdout.cursorTo(0);
if (downloads.length === 0 || !downloads.every(([_label, download]) => typeof download === 'string')) {
setTimeout(() => render(), 1000);
return;
}
process.stdout.moveCursor(0, downloads.length + 1);
}
function setProgress(label, completedBytes, totalBytes, hash) {
const memory = getMemoryUsage();
stats.peakMemoryUsage = Math.max(memory, stats.peakMemoryUsage);
stats.downloads[label] = hash || Math.round((completedBytes / totalBytes) * 100);
}
async function buffered(label) {
const hash = new blake2.Hash('blake2b');
const imageRes = await bhttp.get(file, {
onDownloadProgress(completedBytes, totalBytes) {
setProgress(label, completedBytes, totalBytes);
},
});
hash.update(imageRes.body);
setProgress(label, null, null, hash.digest('hex'));
await fsPromises.writeFile(`/mnt/stor/Pictures/traxxx/temp/buffered-${label}.bin`, imageRes.body);
}
async function streamed(label) {
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const hashStream = new PassThrough();
const targetStream = fs.createWriteStream(`/mnt/stor/Pictures/traxxx/temp/streamed-${label}.bin`);
const imageRes = await bhttp.get(file, {
stream: true,
});
const stream = imageRes
.pipe(hashStream)
.pipe(targetStream);
imageRes.on('progress', (completedBytes, totalBytes) => {
setProgress(label, completedBytes, totalBytes);
});
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
stream.on('finish', () => {
hash.end();
setProgress(label, null, null, hash.read());
});
}
async function init() {
const n = argv.n || 1;
if (argv._.includes('stream')) {
console.log('using streams');
render();
await Promise.map(Array.from({ length: n }), async (value, index) => streamed(index + 1));
return;
}
if (argv._.includes('buffer')) {
console.log('using buffers');
render();
await Promise.map(Array.from({ length: n }), async (value, index) => buffered(index + 1));
}
}
init();
-153
View File
@@ -1,153 +0,0 @@
'use strict';
const arrayEqual = require('array-equal');
/* eslint-disable */
function evaluateBranch(tree, modifiers) {
const result = tree.map((expression) => {
if (expression.type === 'group') {
return evaluateBranch(expression.values, expression.modifiers);
} if (expression.modifiers.length === 0) {
return ''; // This is a trigger to stringify the previous values
} if (arrayEqual(['plus'], expression.modifiers)) {
return 0;
} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
return true;
} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
return true;
} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
return true;
} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
return 1;
}
throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
}).reduce((combined, value) => {
if (value === '') {
return combined.toString();
}
if (value === true) {
value = 1;
}
if (typeof combined === 'string') {
return combined + value.toString();
}
return combined + value;
}, 0);
if (modifiers == null) {
return result;
}
if (arrayEqual(['plus'], modifiers)) {
return parseInt(result);
}
return result;
}
function evaluate(tree) {
return evaluateBranch(tree);
}
function parse(string) {
const length = string.length;
let byte;
let stateFinishedItem = false;
const modifierStack = [[]];
const itemStack = [[]];
let currentStack = itemStack[0];
let currentModifiers = modifierStack[0];
let stackLevel = 0;
for (let pos = 0; pos < length; pos++) {
byte = string[pos];
switch (byte) {
case '+':
if (pos === 0 || stateFinishedItem === false) {
// Modifier / number-cast
currentModifiers.push('plus');
stateFinishedItem = false;
} else {
// Addition, we don't need to do anything here
}
break;
case '!':
stateFinishedItem = false;
currentModifiers.push('negate');
break;
case '(':
stateFinishedItem = false;
stackLevel++;
itemStack[stackLevel] = currentStack = [];
modifierStack[stackLevel] = currentModifiers = [];
break;
case ')':
if (stackLevel === 0) {
throw new Error('Encountered ) without matching (');
}
stackLevel--;
stateFinishedItem = true;
currentStack = itemStack[stackLevel];
currentStack.push({
type: 'group',
values: itemStack[stackLevel + 1],
modifiers: modifierStack[stackLevel],
});
currentModifiers = modifierStack[stackLevel] = [];
break;
case '[':
if (string[pos + 1] === ']') {
// Reached the brackets; end of the modifier sequence
currentStack.push({
type: 'brackets',
modifiers: currentModifiers,
});
currentModifiers = [];
pos += 1; // Skip over the closing bracket
stateFinishedItem = true;
} else {
throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
}
}
}
return itemStack[0];
}
function parseExpression(string) {
return evaluate(parse(string));
}
function findAll(regex, target) {
const results = []; let
match;
while (match = regex.exec(target)) {
results.push(match);
}
return results;
}
function cf(testcase) {
let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
operation: match[1],
expression: match[2],
})).map(({ operation, expression }) => ({
operation,
expression: parseExpression(expression),
}));
initialExpression = parseExpression(initialExpression);
return { parent, child, initialExpression, modifyingExpressions };
};
module.exports = cf;
+1 -1
View File
@@ -82,7 +82,7 @@ function convertApi(rawInput, fromOrTo, to) {
return null;
}
const input = decode(rawInput); // remove html entities, e.g. 5' 8&quot; for 5' 8"
const input = typeof rawInput === 'string' ? decode(rawInput) : rawInput; // remove html entities, e.g. 5' 8&quot; for 5' 8"
try {
if (typeof input === 'string' && to === undefined) {
-3
View File
@@ -1,3 +0,0 @@
'use strict';
module.exports = new Map();
-511
View File
@@ -1,511 +0,0 @@
'use strict';
const config = require('config');
const Promise = require('bluebird');
const bhttp = require('bhttp');
const undici = require('undici');
const fs = require('fs').promises;
// const util = require('util');
// const stream = require('stream');
const { pipeline } = require('stream/promises');
const Bottleneck = require('bottleneck');
const { JSDOM, toughCookie } = require('jsdom');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const windows = require('./http-windows');
const logger = require('../logger')(__filename);
const virtualConsole = require('./virtual-console')(__filename);
const argv = require('../argv');
// const pipeline = util.promisify(stream.pipeline);
const limiters = {
bypass: new Bottleneck({
minTime: 1000,
maxConcurrent: 1,
timeout: 60000,
}),
};
const bypassSessions = new Map();
const browsers = new Map();
Promise.config({
cancellation: true,
});
puppeteer.use(StealthPlugin());
const defaultOptions = {
timeout: argv.requestTimeout,
encodeJSON: true,
parse: false,
headers: {
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
},
};
function useProxy(url) {
if (!config.proxy.enable) {
return false;
}
const { hostname } = new URL(url);
return config.proxy.hostnames.includes(hostname);
}
function useBrowserBypass(url, options) {
if (!config.bypass.browser.enable) {
return null;
}
const { hostname } = new URL(url);
if (options.bypassBrowser === 'shared') {
return true;
}
if (config.bypass.browser.hostnames.includes(hostname)) {
return true;
}
return false;
}
function useCloudflareBypass(url, options) {
if (!config.bypass.cloudflare.enable) {
return null;
}
const { hostname } = new URL(url);
if (options.bypassCloudflare === 'shared') {
return 'shared';
}
if (options.bypassCloudflare === 'independent') {
return hostname;
}
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
return 'shared';
}
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
return hostname;
}
return null;
}
const propMap = {
media: {
interval: 'mediaInterval',
concurrency: 'mediaConcurrency',
},
};
function getLimiterValue(prop, options, hostname) {
const mappedProp = propMap[options.limits]?.[prop] || prop;
if (typeof argv[mappedProp] !== 'undefined') {
return argv[mappedProp];
}
if (typeof options[prop] !== 'undefined') {
return options[prop];
}
if (config.limits[hostname]?.enable !== false && typeof config.limits[hostname]?.[prop] !== 'undefined') {
return config.limits[hostname][prop];
}
return config.limits[options.limits || 'default'][prop];
}
function getLimiter(options = {}, url) {
const { hostname } = new URL(url);
const interval = getLimiterValue('interval', options, hostname);
const concurrency = getLimiterValue('concurrency', options, hostname);
if (!limiters[interval]?.[concurrency]) {
limiters[interval] = limiters[interval] || {};
limiters[interval][concurrency] = new Bottleneck({
minTime: interval,
maxConcurrent: concurrency,
timeout: (options.timeout || defaultOptions.timeout) + 10000, // timeout 10 seconds after bhttp should
});
}
return limiters[interval][concurrency];
}
function extractJson(body, headers) {
if (headers['content-type'].includes('application/json')) {
const { document } = new JSDOM(body, { virtualConsole }).window;
const dataString = document.querySelector('body > pre')?.textContent;
if (dataString) {
const data = JSON.parse(dataString);
return data;
}
}
return body;
}
async function getBrowserSession(identifier, options = {}) {
return limiters.bypass.schedule(async () => {
const browserIdentifier = options.useGlobalBrowser === false ? identifier : null;
if (!browsers.has(browserIdentifier)) {
const newBrowser = await puppeteer.launch({
headless: typeof options.headless === 'undefined' ? 'new' : options.headless,
args: [
...(options.useProxy && config.proxy.enabled ? [`--proxy-server=${config.proxy.host}:${config.proxy.port}`] : []),
],
// headless: false,
});
browsers.set(browserIdentifier, newBrowser);
logger.info('Initialized puppeteer browser');
}
const browser = browsers.get(browserIdentifier);
const tab = await browser.newPage();
logger.verbose(`Opened puppeteer tab${identifier ? ` for ${identifier}` : ''}`);
return { browser, tab };
});
}
async function bypassBrowserRequest(url, options) {
const { tab } = await getBrowserSession(new URL(url).hostname, options);
const res = await tab.goto(url);
if (options.bypass?.delay) {
await Promise.delay(options.bypass.delay);
}
if (typeof options.bypass?.evaluate === 'function') {
await tab.evaluate(options.bypass.evaluate, options.bypass);
}
const rawBody = await tab.content();
const headers = res.headers();
const body = extractJson(rawBody, headers);
const statusCode = res.status();
if (!statusCode === 200) {
throw new Error(`Puppeteer bypass failed for ${url} (${statusCode}): ${body?.message}`);
}
return {
body,
statusCode,
headers,
};
}
async function getBypassSession(url, hostname) {
if (bypassSessions.has(hostname)) {
return bypassSessions.get(hostname);
}
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.create',
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
});
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
}
bypassSessions.set(hostname, sessionRes.body.session);
return sessionRes.body.session;
}
async function destroyBypassSession(sessionId) {
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.destroy',
session: sessionId,
}, {
encodeJSON: true,
}));
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
bypassSessions.delete(sessionId);
logger.verbose(`Destroyed bypass session ${sessionId}`);
return true;
}
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
return false;
}
async function destroyBypassSessions() {
if (!config.bypass.cloudflare.enabled) {
return;
}
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.list',
}, {
encodeJSON: true,
}));
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
}
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
}
async function destroyBrowserSessions() {
await Promise.all(Array.from(browsers.values(), async (browser) => browser?.close()));
}
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: `request.${method}`,
url,
session: sessionId,
maxTimeout: options.timeout,
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
}));
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
if (/session closed/i.test(res.body?.message) && attempts < 3) {
await destroyBypassSession(sessionId);
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
}
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
}
const resBody = extractJson(res.body.solution.response, res.body.solution.headers);
return {
body: resBody,
statusCode: res.body.solution.status,
headers: res.body.solution.headers,
};
}
const defaultAgent = new undici.Agent({
allowH2: true,
connect: {
rejectUnauthorized: false,
},
});
const proxyAgent = new undici.ProxyAgent({
uri: `http://${config.proxy.host}:${config.proxy.port}`,
});
async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
const withProxy = useProxy(url);
const withBrowserBypass = useBrowserBypass(url, requestOptions);
const withCloudflareBypass = useCloudflareBypass(url, requestOptions);
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
if (withBrowserBypass) {
if (method !== 'get') throw new Error('Browser bypass only supports GET');
return bypassBrowserRequest(url, requestOptions);
}
if (withCloudflareBypass) {
return bypassCloudflareRequest(url, method, body, withCloudflareBypass, requestOptions);
}
const headers = {
...requestOptions.headers,
};
const res = await undici.request(url, {
method: method.toUpperCase(),
headers,
body: body ?? null,
dispatcher: withProxy
? proxyAgent
: defaultAgent,
maxRedirections: 0, // handle manually
});
if (res.headers.location && redirects < 3) {
// Drain the body to free the socket before redirecting
await res.body.dump();
const nextUrl = new URL(res.headers.location, url).href;
return request(method, nextUrl, body, requestOptions, limiter, redirects + 1);
}
return res;
}
async function finalizeResult(res, url, options) {
if (options.destination) {
await pipeline(
res.body,
...(options.transforms || []),
options.destination,
);
return {
statusCode: res.statusCode,
headers: res.headers,
status: res.statusCode,
ok: res.statusCode >= 200 && res.statusCode <= 299,
};
}
const buffer = await res.body.arrayBuffer();
const html = Buffer.from(buffer).toString();
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
const pathname = new URL(url).pathname.replace(/\//g, '_');
if (window && /fetchScene|fetchMovie/.test(new Error().stack)) {
windows.set(pathname, window);
}
if (argv.saveHtml) {
await fs.writeFile(`./html/${pathname}.html`, html);
}
return {
statusCode: res.statusCode,
status: res.statusCode,
headers: res.headers,
body: html,
html,
document: window?.document || null,
window,
ok: res.statusCode >= 200 && res.statusCode <= 299,
};
}
function getTimeout(options, url) {
return new Promise((_resolve, reject, onCancel) => {
const timeout = setTimeout(() => {
logger.debug(`Canceled timed out request to ${url}`);
reject(new Error(`URL ${url} timed out`));
}, (options?.timeout || defaultOptions.timeout) + 10000);
onCancel(() => {
clearTimeout(timeout);
});
});
}
async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
if (typeof url !== 'string') {
console.trace(`Bad URL: ${JSON.stringify(url)}`);
}
const options = {
...defaultOptions,
...requestOptions,
headers: {
...Object.fromEntries(requestOptions.includeDefaultHeaders === false ? {} : Object.entries(defaultOptions.headers).map(([key, value]) => [key.toLowerCase(), value])),
...Object.fromEntries(Object.entries(requestOptions.headers || {}).map(([key, value]) => [key.toLowerCase(), value])),
},
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
stream: !!requestOptions.destination,
};
const limiter = getLimiter(options, url);
const timeout = getTimeout(options, url);
const result = await limiter.schedule(async () => Promise.race([request(method, url, body, options, limiter), timeout]));
timeout.cancel();
const curatedResult = await finalizeResult(result, url, options);
logger.silly(`Response ${curatedResult.status} for ${method.toUpperCase()} ${url}`);
return curatedResult;
}
async function get(url, options) {
return scheduleRequest('get', url, null, options);
}
async function post(url, body, options) {
return scheduleRequest('post', url, body, options);
}
async function put(url, body, options) {
return scheduleRequest('put', url, body, options);
}
async function patch(url, body, options) {
return scheduleRequest('patch', url, body, options);
}
async function del(url, options) {
return scheduleRequest('delete', url, null, options);
}
async function head(url, options) {
return scheduleRequest('head', url, null, options);
}
function getSession(options) {
return bhttp.session({ ...defaultOptions, ...options });
}
function getCookieJar(store, options) {
return new toughCookie.CookieJar(store, {
looseMode: true,
...options,
});
}
module.exports = {
toughCookie,
get,
head,
post,
delete: del,
put,
patch,
session: getSession,
cookieJar: getCookieJar,
browsers,
getBrowserSession,
getBypassSession,
getSession,
getCookieJar,
destroyBypassSessions,
destroyBrowserSessions,
proxyAgent,
};
-18
View File
@@ -1,18 +0,0 @@
'use strict';
const bhttp = require('bhttp');
const sharp = require('sharp');
const { argv } = require('yargs');
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
async function scan() {
console.log(url);
const res = await bhttp.get(url);
const stats = await sharp(res.body).stats();
console.log(stats);
}
scan();
-21
View File
@@ -1,21 +0,0 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const knex = require('../knex');
async function run() {
const network = await knex('networks').where('slug', 'mofos').first();
const sites = await knex('sites').where('network_id', network.id);
await Promise.map(sites, async (site) => {
const res = await bhttp.get(site.url);
console.log(site.url, res.statusCode);
}, {
concurrency: 5,
});
}
run();
-37
View File
@@ -1,37 +0,0 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const fs = require('fs-extra');
const knex = require('../knex');
async function init() {
const sites = await knex('sites')
.select('networks.name', 'sites.slug')
.join('networks', 'networks.id', 'sites.network_id')
.where('networks.slug', 'score');
await Promise.map(sites, async (site) => {
const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
console.log(url);
const res = await bhttp.get(url, {
responseTimeout: 5000,
});
if (res.statusCode === 200) {
console.log(`Saving logo for ${site.slug}`);
await fs.writeFile(`./score/${site.slug}.png`, res.body);
}
console.log(`No logo found for ${site.slug}`);
}, {
concurrency: 10,
});
knex.destroy();
}
init();
-55
View File
@@ -1,55 +0,0 @@
'use strict';
const config = require('config');
const { PassThrough } = require('stream');
const fs = require('fs');
const path = require('path');
const bhttp = require('bhttp');
const blake2 = require('blake2');
const sharp = require('sharp');
const url = 'https://thumbs.julesjordan.com/trial/content//upload/dl03/julesjordan/oil_overload_16_scene2//photos/alina_lopez_jules_jordan_com_77.jpg';
async function init() {
const hash = new blake2.Hash('blake2b');
hash.setEncoding('hex');
const res = await bhttp.get(url, {
stream: true,
});
const metaStream = sharp();
const hashStream = new PassThrough();
const target = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina.jpg'));
const thumbTarget = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina_thumb.jpg'));
hashStream.on('data', (chunk) => {
hash.write(chunk);
});
metaStream.clone()
.resize(320)
.pipe(thumbTarget);
const stream = res
.pipe(metaStream)
.pipe(hashStream)
.pipe(target);
stream.on('finish', () => {
hash.end();
const digest = hash.read();
console.log('stream', digest);
});
metaStream.on('info', (info) => {
console.log('info', info);
});
const stats = await metaStream.stats();
console.log('stats', stats);
}
init();
-827
View File
@@ -1,827 +0,0 @@
const Promise = require('bluebird');
const bhttp = require('bhttp');
const fs = require('fs');
const channels = [{
name: 'Analyzed Girls',
slug: 'analyzedgirls',
url: 'https://analyzedgirls.com',
alias: ['ag',
],
tags: ['anal',
],
parent: 'teencoreclub',
parameters: {
siteId: 178,
},
},
{
name: 'Archived Updates',
slug: 'archivedupdates',
url: 'https://archivedupdates.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 482,
},
},
{
name: 'Ass Teen Mouth',
slug: 'assteenmouth',
url: 'https://assteenmouth.com',
alias: ['atm',
],
tags: ['anal',
'atm',
'teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 180,
},
},
{
name: 'Bang Teen Pussy',
slug: 'bangteenpussy',
url: 'https://bangteenpussy.com',
alias: ['btp',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 182,
},
},
{
name: 'Brutal Invasion',
slug: 'brutalinvasion',
url: 'https://brutalinvasion.com',
alias: ['bin',
],
parent: 'teencoreclub',
parameters: {
siteId: 184,
},
},
{
name: 'Clean My Ass',
slug: 'cleanmyass',
url: 'https://cleanmyass.com',
tags: ['ass-eating'],
alias: ['cma',
],
parent: 'teencoreclub',
parameters: {
siteId: 362,
},
},
{
name: 'College Party Time',
slug: 'collegepartytime',
url: 'https://collegepartytime.com',
alias: ['cpt',
],
parent: 'teencoreclub',
parameters: {
siteId: 278,
},
},
{
name: 'Cumaholic Teens',
slug: 'cumaholicteens',
url: 'https://cumaholicteens.com',
alias: ['ct',
'cumoholic teens',
'cumoholicteens',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 186,
},
},
{
name: 'Cum Filled Throat',
slug: 'cumfilledthroat',
url: 'https://cumfilledthroat.com',
alias: ['cft',
],
parent: 'teencoreclub',
parameters: {
siteId: 280,
},
},
{
name: 'Cum Sumption Cocktail',
slug: 'cumsumptioncocktail',
url: 'https://cumsumptioncocktail.com',
alias: ['csc',
],
parent: 'teencoreclub',
parameters: {
siteId: 282,
},
},
{
name: 'Defiled18',
slug: 'defiled18',
url: 'https://defiled18.com',
alias: ['d18',
],
parent: 'teencoreclub',
parameters: {
siteId: 188,
},
},
{
name: 'Dirty Baby Sitter',
slug: 'dirtybabysitter',
url: 'https://dirtybabysitter.com',
alias: ['dbs',
],
parent: 'teencoreclub',
parameters: {
siteId: 284,
},
},
{
name: 'Double Teamed Teens',
slug: 'doubleteamedteens',
url: 'https://doubleteamedteens.com',
alias: ['dtt',
],
tags: ['teen',
'dp',
],
parent: 'teencoreclub',
parameters: {
siteId: 190,
},
},
{
name: 'Dreamteens HD',
slug: 'dreamteenshd',
url: 'https://dreamteenshd.com',
alias: ['dhd',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 192,
},
},
{
name: 'Drill Her Butt',
slug: 'drillherbutt',
url: 'https://drillherbutt.com',
tags: ['anal'],
alias: ['dhb',
],
parent: 'teencoreclub',
parameters: {
siteId: 288,
},
},
{
name: 'Dual Throat',
slug: 'dualthroat',
url: 'https://dualthroat.com',
alias: ['dt',
],
parent: 'teencoreclub',
parameters: {
siteId: 290,
},
},
{
name: 'Gang Land Victims',
slug: 'ganglandvictims',
url: 'https://ganglandvictims.com',
alias: ['glv',
],
tags: ['gangbang',
],
parent: 'teencoreclub',
parameters: {
siteId: 292,
},
},
{
name: 'Girls Got Cream',
slug: 'girlsgotcream',
url: 'https://girlsgotcream.com',
alias: ['ggc',
],
parent: 'teencoreclub',
parameters: {
siteId: 194,
},
},
{
name: 'Hardcore Youth',
slug: 'hardcoreyouth',
url: 'https://hardcoreyouth.com',
alias: ['hcy',
],
parent: 'teencoreclub',
parameters: {
siteId: 196,
},
},
{
name: 'Little Hellcat',
slug: 'littlehellcat',
url: 'https://littlehellcat.com',
alias: ['lhc',
],
parent: 'teencoreclub',
parameters: {
siteId: 198,
},
},
{
name: 'Little Teen Suckers',
slug: 'littleteensuckers',
url: 'https://littleteensuckers.com',
alias: ['lts',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 294,
},
},
{
name: 'MAIKO',
slug: 'maiko',
url: 'https://maiko.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 566,
},
},
{
name: 'Make Teen Gape',
slug: 'maketeengape',
url: 'https://maketeengape.com',
alias: ['mtg',
],
tags: ['teen',
'gaping',
],
parent: 'teencoreclub',
parameters: {
siteId: 200,
},
},
{
name: 'Make Teen Moan',
slug: 'maketeenmoan',
url: 'https://maketeenmoan.com',
alias: ['mtm',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 296,
},
},
{
name: 'Mega Penetrations',
slug: 'megapenetrations',
url: 'https://megapenetrations.com',
alias: ['mp',
],
parent: 'teencoreclub',
parameters: {
siteId: 298,
},
},
{
name: 'Messy Gang Bangs',
slug: 'messygangbangs',
url: 'https://messygangbangs.com',
alias: ['mgb',
],
tags: ['gangbang',
],
parent: 'teencoreclub',
parameters: {
siteId: 300,
},
},
{
name: 'My Black Coeds',
slug: 'myblackcoeds',
url: 'https://myblackcoeds.com',
alias: ['mbc',
],
tags: ['ebony',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 302,
},
},
{
name: 'My Latina Teen',
slug: 'mylatinateen',
url: 'https://mylatinateen.com',
alias: ['mlt',
],
tags: ['teen',
'latina',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 304,
},
},
{
name: 'Nasty Ass Lickers',
slug: 'nastyasslickers',
url: 'https://nastyasslickers.com',
tags: ['ass-eating'],
alias: ['nal',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 306,
},
},
{
name: 'Naughty Little Nymphs',
slug: 'naughtylittlenymphs',
url: 'https://naughtylittlenymphs.com',
alias: ['nln',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 308,
},
},
{
name: 'NC UK Sinners',
slug: 'ncuksinners',
url: 'https://ncuksinners.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 568,
},
},
{
name: 'NC Verso Cinema',
slug: 'ncversocinema',
url: 'https://ncversocinema.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 570,
},
},
{
name: 'Network Mixed',
slug: 'networkmixed',
url: 'https://networkmixed.com',
alias: ['anc',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 360,
},
},
{
name: 'Never Done That Before',
slug: 'neverdonethatbefore',
url: 'https://neverdonethatbefore.com',
alias: ['ndt',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 310,
},
},
{
name: 'Nylon Sweeties',
slug: 'nylonsweeties',
url: 'https://nylonsweeties.com',
alias: ['nsw',
],
parent: 'teencoreclub',
parameters: {
siteId: 202,
},
},
{
name: 'Pink Eye Sluts',
slug: 'pinkeyesluts',
url: 'https://pinkeyesluts.com',
alias: ['pes',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 312,
},
},
{
name: 'Plug2 Holes',
slug: 'plug2holes',
url: 'https://plug2holes.com',
alias: ['p2h',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 314,
},
},
{
name: 'Pussy Babes',
slug: 'pussybabes',
url: 'https://pussybabes.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 556,
},
},
{
name: 'Road Gang Bangs',
slug: 'roadgangbangs',
url: 'https://roadgangbangs.com',
alias: ['rgb',
],
tags: ['gangbang',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 316,
},
},
{
name: 'Russian Teen Updates',
slug: 'russianteenupdates',
url: 'https://russianteenupdates.com',
tags: ['teen',
'russian',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 418,
},
},
{
name: 'School Bus Chicks',
slug: 'schoolbuschicks',
url: 'https://schoolbuschicks.com',
alias: ['sbc',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 318,
},
},
{
name: 'Seductive18',
slug: 'seductive18',
url: 'https://seductive18.com',
alias: ['s18',
],
parent: 'teencoreclub',
parameters: {
siteId: 204,
},
},
{
name: 'Show Me Gape',
slug: 'showmegape',
url: 'https://showmegape.com',
tags: ['gaping'],
alias: ['smg',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 320,
},
},
{
name: 'Shy Teachers Pet',
slug: 'shyteacherspet',
url: 'https://shyteacherspet.com',
alias: ['stp',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 322,
},
},
{
name: 'Small Tits Hunter',
slug: 'smalltitshunter',
url: 'https://smalltitshunter.com',
alias: ['sth',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 324,
},
},
{
name: 'Spermantino',
slug: 'spermantino',
url: 'https://spermantino.com',
alias: ['spe',
],
parent: 'teencoreclub',
parameters: {
siteId: 366,
},
},
{
name: 'TCC Network Archives',
slug: 'tccnetworkarchives',
url: 'https://tccnetworkarchives.com',
alias: ['all',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 176,
},
},
{
name: 'Teach My Ass',
slug: 'teachmyass',
url: 'https://teachmyass.com',
tags: ['anal'],
alias: ['tma',
],
parent: 'teencoreclub',
parameters: {
siteId: 368,
},
},
{
name: 'Teenagers Going Wild',
slug: 'teenagersgoingwild',
url: 'https://teenagersgoingwild.com',
alias: ['tgw',
],
tags: ['teen',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 326,
},
},
{
name: 'Teen Anal Casting',
slug: 'teenanalcasting',
url: 'https://teenanalcasting.com',
alias: ['tac',
],
tags: ['anal',
'teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 206,
},
},
{
name: 'Teen Drillers',
slug: 'teendrillers',
url: 'https://teendrillers.com',
alias: ['td',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 208,
},
},
{
name: 'Teen Gina',
slug: 'teengina',
url: 'https://teengina.com',
alias: ['tg',
],
tags: ['teen',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 210,
},
},
{
name: 'Teens Love Blacks',
slug: 'teensloveblacks',
url: 'https://teensloveblacks.com',
alias: ['tlb',
],
tags: ['interracial',
'bbc',
'teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 328,
},
},
{
name: 'Teens Natural Way',
slug: 'teensnaturalway',
url: 'https://teensnaturalway.com',
alias: ['tnw',
],
tags: ['teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 212,
},
},
{
name: 'Teens Try Anal',
slug: 'teenstryanal',
url: 'https://teenstryanal.com',
alias: ['tta',
],
tags: ['anal',
'teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 330,
},
},
{
name: 'Teens Try Blacks',
slug: 'teenstryblacks',
url: 'https://teenstryblacks.com',
alias: ['ttb',
],
tags: ['interracial',
'bbc',
'teen',
],
parent: 'teencoreclub',
parameters: {
siteId: 214,
},
},
{
name: 'Teens Want Orgies',
slug: 'teenswantorgies',
url: 'https://teenswantorgies.com',
alias: ['two',
],
tags: ['teen',
'threesome',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 332,
},
},
{
name: 'Try Teens',
slug: 'tryteens',
url: 'https://tryteens.com',
alias: ['tt',
],
tags: ['teen',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 216,
},
},
{
name: 'Tug Job Queens',
slug: 'tugjobqueens',
url: 'https://tugjobqueens.com',
tags: ['handjob'],
alias: ['tjq',
],
parent: 'teencoreclub',
parameters: {
siteId: 334,
},
},
{
name: 'UK Sinners',
slug: 'uksinners',
url: 'https://uksinners.com',
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 558,
},
},
{
name: 'White Box Black Cocks',
slug: 'whiteboxblackcocks',
url: 'https://whiteboxblackcocks.com',
alias: ['wbc',
],
tags: ['interracial',
'bbc',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 336,
},
},
{
name: 'White Teens Black Cocks',
slug: 'whiteteensblackcocks',
url: 'https://whiteteensblackcocks.com',
alias: ['wbc',
],
tags: ['interracial',
'bbc',
'teen',
],
parent: 'teencoreclub',
hasLogo: false,
parameters: {
siteId: 218,
},
},
{
name: 'Young Throats',
slug: 'youngthroats',
url: 'https://youngthroats.com',
tags: ['blowjob', 'deepthroat', 'facefucking'],
alias: ['yt',
],
parent: 'teencoreclub',
parameters: {
siteId: 220,
},
}];
async function init() {
const results = await Promise.map(channels, async (channel) => {
try {
const res = await bhttp.head(channel.url, {
followRedirects: false,
responseTimeout: 30000,
});
console.log(res.statusCode, channel.url);
if (res.statusCode === 200) {
return channel;
}
throw new Error(`${channel.url} no longer available: ${res.statusCode}`);
} catch (error) {
console.error(channel.url, error.message);
return {
name: channel.name,
slug: channel.slug,
url: `https://teencoreclub.com/browsevideos/site/${channel.parameters.siteId}/${channel.name.replace(/\s+/g, '')}`,
...(channel.tags && { tags: channel.tags }),
...(channel.alias && { alias: channel.alias }),
parent: channel.parent,
...(channel.hasLogo !== undefined && { hasLogo: channel.hasLogo }),
parameters: channel.parameters,
};
}
}, { concurrency: 5 });
await fs.promises.writeFile('./tcc.json', JSON.stringify(results, null, 4));
console.log('done!');
}
init();
-18
View File
@@ -1,18 +0,0 @@
'use strict';
const { VirtualConsole } = require('jsdom');
const path = require('path');
const logger = require('../logger')(__filename);
function getVirtualConsole(filepath) {
const virtualConsole = new VirtualConsole();
const context = path.basename(filepath).replace(path.extname(filepath), '');
virtualConsole.on('error', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
virtualConsole.on('jsdomError', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
return virtualConsole;
}
module.exports = getVirtualConsole;
-96
View File
@@ -1,96 +0,0 @@
const webpack = require('webpack');
const path = require('path');
const { VueLoaderPlugin } = require('vue-loader');
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
const autoprefixer = require('autoprefixer');
const ESLintPlugin = require('eslint-webpack-plugin');
module.exports = {
entry: {
bundle: './assets/js/main.js',
},
output: {
filename: '[name].js',
path: path.join(__dirname, 'public/js'),
},
module: {
rules: [
{
test: /\.vue$/,
include: [
path.resolve(__dirname, 'assets'),
],
loader: 'vue-loader',
options: {
preserveWhitespace: false,
},
},
{
test: /\.js$/,
exclude: /node_modules/,
use: [
{
loader: 'babel-loader',
options: {
babelrc: false,
plugins: [
'@babel/plugin-proposal-object-rest-spread',
],
},
},
],
},
{
test: /\.s?css$/,
use: [
MiniCssExtractPlugin.loader,
{
loader: 'css-loader',
options: {
sourceMap: true,
url: false,
},
},
{
loader: 'postcss-loader',
options: {
postcssOptions: {
plugins: [autoprefixer],
},
sourceMap: true,
},
},
{
loader: 'sass-loader',
options: {
sourceMap: true,
additionalData: '@import "assets/css/_breakpoints.scss";',
},
},
],
},
{
test: /\.svg/,
use: 'raw-loader',
},
],
},
plugins: [
new VueLoaderPlugin(),
new ESLintPlugin(),
new MiniCssExtractPlugin({
filename: '../css/style.css',
}),
new webpack.DefinePlugin({
VERSION: JSON.stringify(process.env.npm_package_version),
}),
],
resolve: {
alias: {
theme: path.join(__dirname, 'assets/css/_theme.scss'),
breakpoints: path.join(__dirname, 'assets/css/_breakpoints.scss'),
config: path.join(__dirname, `assets/js/config/${process.env.NODE_ENV || 'default'}.js`),
vue: 'vue/dist/vue.esm-bundler.js',
},
},
};