Compare commits
23
Commits
83c78ade5d
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0d727e91c6 | ||
|
|
6509b2a25a | ||
|
|
171a8c9944 | ||
|
|
87946580f1 | ||
|
|
34d0f31347 | ||
|
|
ecacecfd62 | ||
|
|
1db0833794 | ||
|
|
06163e5ba2 | ||
|
|
da71b8c179 | ||
|
|
55ddacbf0d | ||
|
|
81bed09ced | ||
|
|
8c3f1b0c80 | ||
|
|
732c72c6dd | ||
|
|
43189f401a | ||
|
|
02676efd0c | ||
|
|
12c27722ee | ||
|
|
27be031068 | ||
|
|
b4d07bbaa6 | ||
|
|
06ae62436a | ||
|
|
98121e6162 | ||
|
|
e447a0b6da | ||
|
|
530d45448a | ||
|
|
6a0759693c |
@@ -5,11 +5,12 @@
|
||||
"node": true
|
||||
},
|
||||
"parserOptions": {
|
||||
"parser": "@babel/eslint-parser",
|
||||
"ecmaVersion": 2024,
|
||||
"sourceType": "script"
|
||||
},
|
||||
"rules": {
|
||||
"import/no-unresolved": ["error", {"ignore": ["^yargs/helpers$", "^object-merge-advanced$"]}],
|
||||
"import/extensions": ["error", "ignorePackages", {"pattern": {"js": "never", "mjs": "never", "jsx": "never"}, "pathGroupOverrides": [{"pattern": "@noble/hashes/**", "action": "ignore"}]}],
|
||||
"indent": ["error", "tab"],
|
||||
"no-tabs": "off",
|
||||
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
**NSFW** - The latest releases from your favorite porn studios in one place.
|
||||
|
||||
## Install & run
|
||||
Use [nvm](https://github.com/creationix/nvm) to install NodeJS v16.8.0 or newer. Download and unpack or clone this repository, and run the following in the root directory:
|
||||
Use [nvm](https://github.com/creationix/nvm) to install NodeJS v26.13.0, and make sure it's active (`nvm use`). Download and unpack or clone this repository, and run the following in the root directory:
|
||||
|
||||
`npm install`
|
||||
`npm ci`
|
||||
|
||||
### Set up database
|
||||
Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user.
|
||||
@@ -18,6 +18,11 @@ Do not modify `config/default.js`, but instead create a copy at `config/local.js
|
||||
|
||||
You can also use `npm run flush` to run both steps at once, and wipe the database completely later.
|
||||
|
||||
### Install browser
|
||||
Unless configured to use unprint's remote browser server, a browser needs to be installed locally:
|
||||
|
||||
`npx patchright install chromium`
|
||||
|
||||
#### Networks and channels
|
||||
To scrape the networks and channels available in the database, you can configure `include` and `exclude` lists. To include all available channels and only use the `exclude` list, leave the `include` parameter unconfigured. The `exclude` lists will exclude channels and child networks from networks on the `include` lists, but not vice versa. That is, if the `include` list includes a network and the `exclude` list excludes one of that network's channels, the channel will not be scraped. However, if the `include` list includes a channel, and the `exclude` list includes its parent network, the channel will be scraped.
|
||||
|
||||
@@ -50,11 +55,7 @@ exclude: {
|
||||
},
|
||||
```
|
||||
|
||||
### Building
|
||||
To build traxxx, run the following command:
|
||||
|
||||
`npm run build`
|
||||
|
||||
### Thumbnails
|
||||
To generate thumbnails for new logos and tag photos, install ImageMagick and run:
|
||||
|
||||
`npm run logos-thumbs`
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
|
||||
const scenesFields = `
|
||||
id int,
|
||||
title text,
|
||||
title_filtered text,
|
||||
shoot_id text,
|
||||
channel_id int,
|
||||
channel_name text,
|
||||
channel_slug text,
|
||||
network_id int,
|
||||
network_name text,
|
||||
network_slug text,
|
||||
studio_id int,
|
||||
studio_name text,
|
||||
studio_slug text,
|
||||
entity_ids multi,
|
||||
actor_ids multi,
|
||||
actors text,
|
||||
tag_ids multi,
|
||||
tags text,
|
||||
movie_ids multi,
|
||||
movies text,
|
||||
serie_ids multi,
|
||||
series text,
|
||||
meta text,
|
||||
date timestamp,
|
||||
is_showcased bool,
|
||||
created_at timestamp,
|
||||
effective_date timestamp,
|
||||
stashed int,
|
||||
dupe_index int
|
||||
`;
|
||||
|
||||
const moviesFields = `
|
||||
id int,
|
||||
title text,
|
||||
title_filtered text,
|
||||
channel_id int,
|
||||
channel_name text,
|
||||
channel_slug text,
|
||||
network_id int,
|
||||
network_name text,
|
||||
network_slug text,
|
||||
entity_ids multi,
|
||||
actor_ids multi,
|
||||
actors text,
|
||||
tag_ids multi,
|
||||
tags text,
|
||||
meta text,
|
||||
date timestamp,
|
||||
has_cover bool,
|
||||
created_at timestamp,
|
||||
effective_date timestamp,
|
||||
stashed int,
|
||||
stashed_scenes int,
|
||||
stashed_total int,
|
||||
dupe_index int
|
||||
`;
|
||||
|
||||
const actorsFields = `
|
||||
id int,
|
||||
name text,
|
||||
slug string,
|
||||
gender string,
|
||||
date_of_birth timestamp,
|
||||
country string,
|
||||
has_avatar bool,
|
||||
mass int,
|
||||
height int,
|
||||
cup string,
|
||||
natural_boobs int,
|
||||
penis_length int,
|
||||
penis_girth int,
|
||||
stashed int,
|
||||
scenes int
|
||||
`;
|
||||
|
||||
exports.up = async () => {
|
||||
try {
|
||||
await utilsApi.sql(`create table if not exists scenes (${scenesFields})`);
|
||||
|
||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
||||
scene_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql(`create table if not exists movies (${moviesFields})`);
|
||||
|
||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
||||
movie_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`);
|
||||
|
||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
||||
actor_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
/*
|
||||
await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_movies', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_actors', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_series', (table) => table.increments('id'));
|
||||
*/
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
};
|
||||
|
||||
exports.down = async (knex) => {
|
||||
await utilsApi.sql('drop table if exists scenes');
|
||||
await utilsApi.sql('drop table if exists scenes_stashed');
|
||||
await utilsApi.sql('drop table if exists movies');
|
||||
await utilsApi.sql('drop table if exists movies_stashed');
|
||||
await utilsApi.sql('drop table if exists actors');
|
||||
await utilsApi.sql('drop table if exists actors_stashed');
|
||||
|
||||
await knex.schema.alterTable('stashes_scenes', (table) => table.dropColumn('id'));
|
||||
await knex.schema.alterTable('stashes_movies', (table) => table.dropColumn('id'));
|
||||
await knex.schema.alterTable('stashes_actors', (table) => table.dropColumn('id'));
|
||||
await knex.schema.alterTable('stashes_series', (table) => table.dropColumn('id'));
|
||||
};
|
||||
@@ -216,12 +216,7 @@ module.exports = {
|
||||
},
|
||||
bypass: {
|
||||
browser: {
|
||||
enable: false,
|
||||
clientRetirement: 20,
|
||||
hostnames: [ // these can run in the same browser session
|
||||
'www.kink.com',
|
||||
'store2.psmcdn.net', // Team Skeet API
|
||||
],
|
||||
remote: {
|
||||
enable: false,
|
||||
use: false,
|
||||
@@ -229,15 +224,6 @@ module.exports = {
|
||||
key: null,
|
||||
},
|
||||
},
|
||||
cloudflare: {
|
||||
enable: false,
|
||||
auto: true, // try bypass when CF challenge is detected
|
||||
path: 'http://localhost:8191/v1',
|
||||
sharedHostnames: [ // these can run in the same browser session
|
||||
'store2.psmcdn.net', // Team Skeet API
|
||||
],
|
||||
independentHostnames: [], // these must run in their own browser session
|
||||
},
|
||||
},
|
||||
limits: {
|
||||
default: {
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
const config = require('config');
|
||||
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: 'traxxx',
|
||||
script: 'src/init.js',
|
||||
exec_mode: 'cluster',
|
||||
instances: 2,
|
||||
restart_delay: 3000,
|
||||
args: '--server',
|
||||
merge_logs: true,
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
},
|
||||
},
|
||||
...(config.analytics.enabled ? [{
|
||||
name: 'umami',
|
||||
script: 'npm start',
|
||||
cwd: process.env.UMAMI_DIR || '../umami',
|
||||
restart_delay: 3000,
|
||||
merge_logs: true,
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
},
|
||||
}] : []),
|
||||
],
|
||||
};
|
||||
Generated
+4570
-18300
File diff suppressed because it is too large
Load Diff
+30
-108
@@ -1,16 +1,13 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.256.3",
|
||||
"version": "1.257.6",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"engines": {
|
||||
"node": ">=26"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node -r source-map-support/register src/init.js",
|
||||
"webpack": "webpack --env=production --mode=production",
|
||||
"webpack-dev": "webpack --env=development --mode=development",
|
||||
"webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
|
||||
"babel": "babel src --source-maps -d dist",
|
||||
"babel-watch": "babel src -w --source-maps -d dist",
|
||||
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
|
||||
"eslint": "eslint src/",
|
||||
"eslint-watch": "esw --watch src/",
|
||||
"knex": "knex",
|
||||
@@ -38,135 +35,60 @@
|
||||
"author": "DebaucheryLibrarian",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.23.4",
|
||||
"@babel/core": "^7.23.3",
|
||||
"@babel/eslint-parser": "^7.23.3",
|
||||
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
|
||||
"@babel/preset-env": "^7.23.3",
|
||||
"@babel/register": "^7.22.15",
|
||||
"autoprefixer": "^10.4.16",
|
||||
"babel-loader": "^9.1.3",
|
||||
"babel-preset-airbnb": "^5.0.0",
|
||||
"css-loader": "^6.8.1",
|
||||
"eslint": "^8.54.0",
|
||||
"eslint-config-airbnb": "^19.0.4",
|
||||
"eslint-config-airbnb-base": "^15.0.0",
|
||||
"eslint-plugin-import": "^2.29.0",
|
||||
"eslint-plugin-jsx-a11y": "^6.8.0",
|
||||
"eslint-plugin-react": "^7.33.2",
|
||||
"eslint-plugin-vue": "^9.18.1",
|
||||
"eslint-watch": "^8.0.0",
|
||||
"eslint-webpack-plugin": "^4.0.1",
|
||||
"mini-css-extract-plugin": "^2.7.6",
|
||||
"node-sass": "^9.0.0",
|
||||
"postcss-loader": "^7.3.3",
|
||||
"raw-loader": "^4.0.2",
|
||||
"sass-loader": "^13.3.2",
|
||||
"style-loader": "^3.3.3",
|
||||
"vue-loader": "^17.3.1",
|
||||
"webpack": "^5.89.0",
|
||||
"webpack-cli": "^5.1.4"
|
||||
"eslint-watch": "^8.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
"@aws-sdk/credential-provider-ini": "^3.458.0",
|
||||
"@aws-sdk/lib-storage": "^3.458.0",
|
||||
"@casl/ability": "^6.5.0",
|
||||
"@faker-js/faker": "^8.3.1",
|
||||
"@graphile-contrib/pg-order-by-related": "^1.0.0",
|
||||
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
||||
"@graphile/pg-aggregates": "^0.1.1",
|
||||
"@faker-js/faker": "^10.6.0",
|
||||
"@noble/hashes": "^2.4.0",
|
||||
"@rsc-parser/react-client": "^1.1.2",
|
||||
"@tensorflow/tfjs-core": "4.22.0",
|
||||
"@tensorflow/tfjs-node": "^4.22.0",
|
||||
"@vladmandic/human": "^3.3.6",
|
||||
"acorn": "^8.11.2",
|
||||
"array-equal": "^1.0.2",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"bhttp": "^1.2.8",
|
||||
"blake2": "^5.0.0",
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
"casual": "^1.6.2",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cli-confirm": "^1.0.1",
|
||||
"clipboardy": "^2.3.0",
|
||||
"cloudscraper": "^4.6.0",
|
||||
"config": "^3.3.9",
|
||||
"connect-session-knex": "^4.0.0",
|
||||
"convert": "^4.14.0",
|
||||
"cookie": "^0.6.0",
|
||||
"css": "^3.0.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"config": "^5.0.1",
|
||||
"convert": "^7.0.2",
|
||||
"csv-stringify": "^6.4.4",
|
||||
"date-fns": "^2.30.0",
|
||||
"dayjs": "^1.11.10",
|
||||
"date-fns": "^4.4.0",
|
||||
"dompurify": "^3.0.6",
|
||||
"ejs": "^3.1.9",
|
||||
"escape-string-regexp": "^4.0.0",
|
||||
"express": "^4.18.2",
|
||||
"express-promise-router": "^4.1.1",
|
||||
"express-query-boolean": "^2.0.0",
|
||||
"express-react-views": "^0.11.0",
|
||||
"express-session": "^1.17.3",
|
||||
"face-api.js": "^0.22.2",
|
||||
"file-type": "^18.7.0",
|
||||
"escape-string-regexp": "^5.0.0",
|
||||
"fluent-ffmpeg": "^2.1.3",
|
||||
"fs-extra": "^11.1.1",
|
||||
"graphile-build": "^4.14.0",
|
||||
"graphile-utils": "^4.14.0",
|
||||
"graphql": "^15.8.0",
|
||||
"html-entities": "^2.4.0",
|
||||
"https-proxy-agent": "^7.0.5",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"inquirer": "^8.2.6",
|
||||
"inquirer": "^14.2.5",
|
||||
"inspector-api": "^1.4.8",
|
||||
"jsdom": "^23.0.1",
|
||||
"jsdom": "^31.0.0",
|
||||
"knex": "^3.0.1",
|
||||
"knex-migrate": "^1.7.4",
|
||||
"longjohn": "^0.2.12",
|
||||
"manticoresearch": "^4.0.0",
|
||||
"merge-anything": "^5.1.7",
|
||||
"mime": "^3.0.0",
|
||||
"mitt": "^3.0.1",
|
||||
"mime": "^4.1.2",
|
||||
"moment": "^2.29.4",
|
||||
"mysql": "^2.18.1",
|
||||
"nanoid": "^3.3.7",
|
||||
"nanoid": "^6.0.2",
|
||||
"node-fetch": "^3.3.2",
|
||||
"npm": "^11.7.0",
|
||||
"object-merge-advanced": "^12.1.0",
|
||||
"object-merge-advanced": "^14.2.5",
|
||||
"object.omit": "^3.0.0",
|
||||
"pg": "^8.11.3",
|
||||
"playwright": "^1.57.0",
|
||||
"playwright-core": "^1.57.0",
|
||||
"postgraphile": "^4.14.0",
|
||||
"postgraphile-plugin-connection-filter": "^2.3.0",
|
||||
"promise-task-queue": "^1.2.0",
|
||||
"prop-types": "^15.8.1",
|
||||
"puppeteer": "^24.4.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"redis": "^4.6.11",
|
||||
"sharp": "^0.33.0-rc.2",
|
||||
"showdown": "^2.1.0",
|
||||
"redis": "^6.3.1",
|
||||
"sharp": "^0.35.5",
|
||||
"source-map-support": "^0.5.21",
|
||||
"template-format": "^1.2.5",
|
||||
"tippy.js": "^6.3.7",
|
||||
"tough-cookie": "^4.1.3",
|
||||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^7.24.7",
|
||||
"unprint": "^0.19.20",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
"videojs-vr": "^2.0.0",
|
||||
"vue": "^3.3.9",
|
||||
"vue-router": "^4.2.5",
|
||||
"vuex": "^4.1.0",
|
||||
"why-is-node-running": "^2.2.2",
|
||||
"template-format": "^2.0.0",
|
||||
"undici": "^8.12.4",
|
||||
"unprint": "^1.0.0-beta.2",
|
||||
"why-is-node-running": "^3.2.3",
|
||||
"winston": "^3.11.0",
|
||||
"winston-daily-rotate-file": "^4.7.1",
|
||||
"yargs": "^17.7.2"
|
||||
"winston-daily-rotate-file": "^5.0.0",
|
||||
"yargs": "^18.2.0"
|
||||
},
|
||||
"allowScripts": {
|
||||
"sharp": true,
|
||||
"@tensorflow/tfjs-node": true,
|
||||
"core-js": false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ const networks = [
|
||||
{
|
||||
slug: 'familystrokes',
|
||||
name: 'Family Strokes',
|
||||
url: 'https://www.familystrokes.com/',
|
||||
url: 'https://www.familystrokes.com',
|
||||
parent: 'paperstreetmedia',
|
||||
parameters: {
|
||||
endpoint: 'familybundle',
|
||||
|
||||
+6
-24
@@ -6027,56 +6027,41 @@ const sites = [
|
||||
slug: 'sexuallybroken',
|
||||
name: 'Sexually Broken',
|
||||
alias: ['seb'],
|
||||
url: 'https://www.sexuallybroken.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=sexuallybroken.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.sexuallybroken.com/sb',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'infernalrestraints',
|
||||
name: 'Infernal Restraints',
|
||||
alias: ['infr'],
|
||||
url: 'https://www.infernalrestraints.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=infernalrestraints.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.infernalrestraints.com/ir',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'hardtied',
|
||||
name: 'Hardtied',
|
||||
alias: ['ht'],
|
||||
url: 'https://www.hardtied.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=hardtied.com',
|
||||
tags: ['bdsm'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.hardtied.com/ht',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'realtimebondage',
|
||||
name: 'Real Time Bondage',
|
||||
alias: ['rtb'],
|
||||
url: 'https://www.realtimebondage.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=realtimebondage.com',
|
||||
tags: ['bdsm', 'live'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.realtimebondage.com/rtb',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'topgrl',
|
||||
name: 'TopGrl',
|
||||
alias: ['tg'],
|
||||
url: 'https://www.topgrl.com',
|
||||
url: 'https://www.insex.com/is/home.php?d=topgrl.com',
|
||||
tags: ['bdsm', 'femdom'],
|
||||
parent: 'insex',
|
||||
parameters: {
|
||||
latest: 'https://www.topgrl.com/tg',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'paintoy',
|
||||
@@ -8578,12 +8563,9 @@ const sites = [
|
||||
{
|
||||
slug: 'mylfdom',
|
||||
name: 'MYLF Dom',
|
||||
url: 'https://www.mylfdom.com',
|
||||
url: 'https://www.mylf.com/series/mylfdom-ts',
|
||||
tags: ['bdsm', 'milf'],
|
||||
parent: 'mylf',
|
||||
parameters: {
|
||||
endpoint: 'mylf_dom',
|
||||
},
|
||||
},
|
||||
// NAUGHTY AMERICA
|
||||
{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const Promise = require('bluebird');
|
||||
const casual = require('casual');
|
||||
const fs = require('fs-extra');
|
||||
const nanoid = require('nanoid');
|
||||
const { nanoid } = require('nanoid');
|
||||
const stringify = Promise.promisify(require('csv-stringify'));
|
||||
const path = require('path');
|
||||
|
||||
|
||||
+3
-7
@@ -4,11 +4,11 @@ const config = require('config');
|
||||
const util = require('util');
|
||||
const Promise = require('bluebird');
|
||||
const moment = require('moment');
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
const DOMPurify = require('dompurify');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const omit = require('object.omit');
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
const unprint = require('unprint');
|
||||
|
||||
const { window } = new JSDOM('');
|
||||
@@ -413,11 +413,7 @@ async function curateProfile(profile, actor) {
|
||||
|
||||
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
|
||||
|
||||
const hasher = curatedProfile.description && blake2
|
||||
.createHash('blake2b', { digestLength: 24 })
|
||||
.update(Buffer.from(slugify(curatedProfile.description)));
|
||||
|
||||
curatedProfile.descriptionHash = curatedProfile.description && hasher.digest('hex');
|
||||
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(Buffer.from(slugify(curatedProfile.description)), { dkLen: 24 })).toString('hex');
|
||||
|
||||
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
||||
|
||||
|
||||
+12
-8
@@ -5,13 +5,12 @@ const startTime = new Date();
|
||||
const config = require('config');
|
||||
const util = require('util');
|
||||
const unprint = require('unprint');
|
||||
const log = require('why-is-node-running');
|
||||
const log = require('why-is-node-running').default;
|
||||
const Inspector = require('inspector-api');
|
||||
const fs = require('fs').promises;
|
||||
const { format, intervalToDuration } = require('date-fns');
|
||||
|
||||
const argv = require('./argv');
|
||||
const http = require('./utils/http');
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
@@ -34,7 +33,7 @@ const unprintOptions = {
|
||||
logErrors: false,
|
||||
timeout: argv.requestTimeout,
|
||||
userAgent: 'traxxx',
|
||||
browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
// browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
||||
apiUserAgent: 'traxxx',
|
||||
clientRetirement: config.bypass.browser.clientRetirement,
|
||||
remote: config.bypass.browser.remote,
|
||||
@@ -210,7 +209,7 @@ async function init() {
|
||||
}
|
||||
|
||||
if (argv.request) {
|
||||
const res = await http[argv.requestMethod](argv.request);
|
||||
const res = await unprint.request(argv.request, { method: argv.requestMethod, interface: 'request' });
|
||||
|
||||
console.log(res.status, res.body);
|
||||
}
|
||||
@@ -238,7 +237,7 @@ async function init() {
|
||||
const movieUrls = (argv.movie || []).concat(moviesFromFile || []);
|
||||
|
||||
const sceneMovies = deepScenes && argv.sceneMovies ? deepScenes.filter((scene) => scene.movie).map((scene) => ({ ...scene.movie, entity: scene.entity })) : [];
|
||||
const deepMovies = argv.sceneMovies || argv.movie || movieUrls ? await fetchMovies([...movieUrls, ...(sceneMovies || []), ...[]]) : sceneMovies;
|
||||
const deepMovies = movieUrls.length > 0 || sceneMovies.length > 0 ? await fetchMovies([...movieUrls, ...sceneMovies]) : sceneMovies;
|
||||
|
||||
const movieScenes = argv.movieScenes ? deepMovies.map((movie) => movie.scenes?.map((scene) => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
|
||||
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
|
||||
@@ -256,9 +255,15 @@ async function init() {
|
||||
|
||||
await associateMovieScenes(storedMovies, allScenes);
|
||||
|
||||
// refreshing the materialized views is slow, skip when nothing was stored (e.g. --request)
|
||||
if (allScenes.length > 0) {
|
||||
await updateSceneSearch(allScenes.map((scene) => scene.id));
|
||||
}
|
||||
|
||||
if (storedMovies.length > 0) {
|
||||
await updateMovieSearch(storedMovies.map((movie) => movie.id));
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Completed in ${Object.entries(intervalToDuration({ start: startTime, end: Date.now() }))
|
||||
.filter(([, value]) => value > 0)
|
||||
@@ -269,12 +274,11 @@ async function init() {
|
||||
logger.error(error);
|
||||
}
|
||||
|
||||
await http.destroyBypassSessions();
|
||||
await http.destroyBrowserSessions();
|
||||
await unprint.closeAllBrowsers();
|
||||
await unprint.closeAgents();
|
||||
|
||||
knex.destroy();
|
||||
redis.disconnect();
|
||||
redis.destroy();
|
||||
|
||||
done = true;
|
||||
}
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
|
||||
const config = require('config');
|
||||
const yargs = require('yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
const moment = require('moment');
|
||||
|
||||
function interpretDate(after, ignoreIfEmpty = false) {
|
||||
@@ -27,7 +28,7 @@ function interpretDate(after, ignoreIfEmpty = false) {
|
||||
.toDate();
|
||||
}
|
||||
|
||||
const { argv } = yargs
|
||||
const { argv } = yargs(hideBin(process.argv))
|
||||
.command('npm start')
|
||||
.command('update', 'fetch latest updates', () => {
|
||||
console.log('UPDATING!');
|
||||
|
||||
-110
@@ -1,110 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const util = require('util');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const knex = require('./knex');
|
||||
const { curateUser, fetchUser } = require('./users');
|
||||
const { HttpError } = require('./errors');
|
||||
|
||||
const scrypt = util.promisify(crypto.scrypt);
|
||||
|
||||
async function verifyPassword(password, storedPassword) {
|
||||
const [salt, hash] = storedPassword.split('/');
|
||||
const hashedPassword = (await scrypt(password, salt, 64)).toString('hex');
|
||||
|
||||
if (hashedPassword === hash) {
|
||||
return true;
|
||||
}
|
||||
|
||||
throw new HttpError('Username or password incorrect', 401);
|
||||
}
|
||||
|
||||
async function login(credentials) {
|
||||
if (!config.auth.login) {
|
||||
throw new HttpError('Authentication is disabled', 405);
|
||||
}
|
||||
|
||||
const user = await fetchUser(credentials.username.trim(), true);
|
||||
|
||||
if (!user) {
|
||||
throw new HttpError('Username or password incorrect', 401);
|
||||
}
|
||||
|
||||
await verifyPassword(credentials.password, user.password);
|
||||
|
||||
await knex('users')
|
||||
.update('last_login', 'NOW()')
|
||||
.where('id', user.id);
|
||||
|
||||
return curateUser(user);
|
||||
}
|
||||
|
||||
async function signup(credentials) {
|
||||
if (!config.auth.signup) {
|
||||
throw new HttpError('Authentication is disabled', 405);
|
||||
}
|
||||
|
||||
const curatedUsername = credentials.username.trim();
|
||||
|
||||
if (!curatedUsername) {
|
||||
throw new HttpError('Username required', 400);
|
||||
}
|
||||
|
||||
if (curatedUsername.length < config.auth.usernameLength[0]) {
|
||||
throw new HttpError('Username is too short', 400);
|
||||
}
|
||||
|
||||
if (curatedUsername.length > config.auth.usernameLength[1]) {
|
||||
throw new HttpError('Username is too long', 400);
|
||||
}
|
||||
|
||||
if (!config.auth.usernamePattern.test(curatedUsername)) {
|
||||
throw new HttpError('Username contains invalid characters', 400);
|
||||
}
|
||||
|
||||
if (!credentials.email) {
|
||||
throw new HttpError('E-mail required', 400);
|
||||
}
|
||||
|
||||
if (credentials.password?.length < 3) {
|
||||
throw new HttpError('Password must be 3 characters or longer', 400);
|
||||
}
|
||||
|
||||
const existingUser = await knex('users')
|
||||
.where('username', curatedUsername)
|
||||
.orWhere('email', credentials.email)
|
||||
.first();
|
||||
|
||||
if (existingUser) {
|
||||
throw new HttpError('Username or e-mail already in use', 409);
|
||||
}
|
||||
|
||||
const salt = crypto.randomBytes(16).toString('hex');
|
||||
const hashedPassword = (await scrypt(credentials.password, salt, 64)).toString('hex');
|
||||
const storedPassword = `${salt}/${hashedPassword}`;
|
||||
|
||||
const [{ id: userId }] = await knex('users')
|
||||
.insert({
|
||||
username: curatedUsername,
|
||||
email: credentials.email,
|
||||
password: storedPassword,
|
||||
})
|
||||
.returning('id');
|
||||
|
||||
await knex('stashes').insert({
|
||||
user_id: userId,
|
||||
name: 'Favorites',
|
||||
slug: 'favorites',
|
||||
public: false,
|
||||
primary: true,
|
||||
});
|
||||
|
||||
return fetchUser(userId);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
login,
|
||||
signup,
|
||||
};
|
||||
+7
-1
@@ -3,7 +3,13 @@
|
||||
const config = require('config');
|
||||
const path = require('path');
|
||||
const fsPromises = require('fs').promises;
|
||||
const Human = require('@vladmandic/human').default;
|
||||
const util = require('util');
|
||||
|
||||
// tfjs-node still uses util helpers that were removed (isNullOrUndefined) or deprecated with a runtime warning (isArray) in Node 23+, must be set before it loads
|
||||
util.isNullOrUndefined ??= (value) => value === null || value === undefined;
|
||||
util.isArray = Array.isArray;
|
||||
|
||||
const Human = require('@vladmandic/human').default; // eslint-disable-line import/order
|
||||
|
||||
const knex = require('./knex');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
-10
@@ -10,7 +10,6 @@ const include = require('./utils/argv-include')(argv);
|
||||
const { fetchReleaseEntities, urlToHostname } = require('./entities');
|
||||
const logger = require('./logger')(__filename);
|
||||
const getRecursiveParameters = require('./utils/get-recursive-parameters');
|
||||
const windows = require('./utils/http-windows');
|
||||
|
||||
const waitImmediate = util.promisify(setImmediate);
|
||||
|
||||
@@ -142,19 +141,10 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
|
||||
? await fetchScene(layoutScraper, baseRelease.url, entity, baseRelease, options)
|
||||
: await fetchMovie(layoutScraper, baseRelease.url, entity, baseRelease, options);
|
||||
|
||||
const pathname = baseRelease.path || (baseRelease.url && new URL(baseRelease.url).pathname.replace(/\//g, '_'));
|
||||
|
||||
if (rawScrapedRelease) {
|
||||
delete rawScrapedRelease.query; // some scrapers pass the qu-wrapped window instance to parent scrapers, filling up memory
|
||||
}
|
||||
|
||||
if (windows.has(pathname)) {
|
||||
logger.debug(`Closing window for ${pathname}`);
|
||||
|
||||
windows.get(pathname).close();
|
||||
windows.delete(pathname);
|
||||
}
|
||||
|
||||
await waitImmediate;
|
||||
|
||||
// logger.debug(`Memory usage after: ${process.memoryUsage.rss() / 1000000} MB (${baseRelease.url})`);
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const argv = require('./argv');
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
require('babel-polyfill');
|
||||
const init = require('./app');
|
||||
|
||||
init();
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
module.exports = {
|
||||
searchApi: new manticore.SearchApi(mantiClient),
|
||||
indexApi: new manticore.IndexApi(mantiClient),
|
||||
utilsApi: new manticore.UtilsApi(),
|
||||
};
|
||||
+23
-12
@@ -8,11 +8,11 @@ const fsPromises = require('fs').promises;
|
||||
const path = require('path');
|
||||
const stream = require('stream');
|
||||
const { nanoid } = require('nanoid/non-secure');
|
||||
const mime = require('mime');
|
||||
const mime = require('mime').default;
|
||||
// const fileType = require('file-type');
|
||||
const ffmpeg = require('fluent-ffmpeg');
|
||||
const sharp = require('sharp');
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
const { format } = require('date-fns');
|
||||
const taskQueue = require('promise-task-queue');
|
||||
const unprint = require('unprint');
|
||||
@@ -23,13 +23,15 @@ const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
|
||||
const logger = require('./logger')(__filename);
|
||||
const argv = require('./argv');
|
||||
const knex = require('./knex');
|
||||
const http = require('./utils/http');
|
||||
const batchInsert = require('./utils/batch-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const { fetchEntityReleaseIds } = require('./entity-releases');
|
||||
|
||||
const streamQueue = taskQueue();
|
||||
|
||||
// carried over from the legacy HTTP module, unprint's raw request interface would identify as traxxx
|
||||
const mediaUserAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36';
|
||||
|
||||
const s3 = new S3Client({
|
||||
region: 'eu-central-1',
|
||||
endpoint: 'https://s3.eu-central-1.wasabisys.com',
|
||||
@@ -637,24 +639,28 @@ async function storeFile(media, options) {
|
||||
}
|
||||
|
||||
async function fetchHttpSource(source, tempFileTarget, hashStream) {
|
||||
const res = await http.get(source.src, {
|
||||
limits: 'media',
|
||||
const res = await unprint.get(source.src, {
|
||||
interface: 'request',
|
||||
limiter: 'media',
|
||||
headers: {
|
||||
'user-agent': mediaUserAgent,
|
||||
// explicit host not allowed in HTTP/2
|
||||
// host: new URL(source.src).hostname,
|
||||
// ...(source.host && { host: source.host }),
|
||||
...(source.referer && { referer: source.referer }),
|
||||
},
|
||||
stream: true, // sources are fetched in parallel, don't gobble up memory
|
||||
followRedirects: source.followRedirects,
|
||||
tls: { rejectUnauthorized: false },
|
||||
// sources are fetched in parallel, stream to file instead of gobbling up memory
|
||||
transforms: [hashStream],
|
||||
destination: tempFileTarget,
|
||||
...(source.interval && { interval: source.interval }),
|
||||
...(source.concurrency && { concurrency: source.concurrency }),
|
||||
// undefined would override the defaults
|
||||
...(typeof source.followRedirects === 'boolean' && { followRedirects: source.followRedirects }),
|
||||
...((argv.mediaInterval ?? source.interval) && { interval: argv.mediaInterval ?? source.interval }),
|
||||
...((argv.mediaConcurrency ?? source.concurrency) && { concurrency: argv.mediaConcurrency ?? source.concurrency }),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Response ${res.status} not OK`);
|
||||
throw new Error(`Response ${res.status} not OK${res.statusText ? ` (${res.statusText})` : ''}`); // status 200 if stream was interrupted
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -723,7 +729,7 @@ async function fetchSource(source, baseMedia) {
|
||||
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
|
||||
let tempFileTarget;
|
||||
|
||||
const hasher = blake2.createHash('blake2b', { digestLength: 24 });
|
||||
const hasher = blake2b.create({ dkLen: 24 });
|
||||
|
||||
let size = 0;
|
||||
|
||||
@@ -743,7 +749,7 @@ async function fetchSource(source, baseMedia) {
|
||||
? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
|
||||
: await fetchHttpSource(source, tempFileTarget, hashStream);
|
||||
|
||||
const hash = hasher.digest('hex');
|
||||
const hash = Buffer.from(hasher.digest()).toString('hex');
|
||||
|
||||
const [type, subtype] = mimetype.split('/');
|
||||
const extension = mime.getExtension(mimetype);
|
||||
@@ -772,6 +778,8 @@ async function fetchSource(source, baseMedia) {
|
||||
hashStream.destroy();
|
||||
tempFileTarget?.destroy();
|
||||
|
||||
await fsPromises.unlink(tempFilePath).catch(() => { /* probably never created */ });
|
||||
|
||||
if (error.code !== 'VERIFY_TYPE') {
|
||||
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
|
||||
|
||||
@@ -914,6 +922,9 @@ async function storeMedias(baseMedias, options) {
|
||||
async (baseMedia) => storeFile(baseMedia, options),
|
||||
{ concurrency: 100 }, // don't overload disk
|
||||
);
|
||||
} else {
|
||||
// duplicates aren't stored, clean up their fetched temp files
|
||||
await Promise.all(existingHashMedias.map(async (media) => media.file?.path && fsPromises.unlink(media.file.path).catch(() => { /* probably already removed */ })));
|
||||
}
|
||||
|
||||
const newMediaWithEntries = savedMedias.filter(Boolean).map((media, index) => curateMediaEntry(media, index));
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
|
||||
+12
-13
@@ -62,7 +62,7 @@ function scrapeAll(scenes, entity) {
|
||||
release.title = query.content('a > span.block');
|
||||
release.date = query.date('a + div', 'MMM DD, YYYY');
|
||||
|
||||
release.duration = query.duration('[data-videopreview-target="duration"]', { attribute: 'data-content' });
|
||||
release.duration = query.duration('[data-videopreview-target="duration"], .video_duration', { attribute: 'data-content' });
|
||||
|
||||
release.actors = query.elements('a + div a[href*="/pornstar"]').map((el) => ({
|
||||
name: unprint.query.content(el),
|
||||
@@ -70,9 +70,11 @@ function scrapeAll(scenes, entity) {
|
||||
}));
|
||||
|
||||
const poster = query.img('img[data-videopreview-target="image"]');
|
||||
const posterUrl = new URL(poster);
|
||||
|
||||
if (poster) {
|
||||
try {
|
||||
const posterUrl = new URL(poster);
|
||||
|
||||
release.poster = [
|
||||
`${posterUrl.origin}${posterUrl.pathname}`,
|
||||
posterUrl.href,
|
||||
@@ -82,18 +84,15 @@ function scrapeAll(scenes, entity) {
|
||||
'application/octet-stream': 'image/jpeg',
|
||||
},
|
||||
}));
|
||||
} catch (error) {
|
||||
// nothing needed
|
||||
}
|
||||
}
|
||||
|
||||
const videoData = query.json('a', { attribute: 'data-videopreview-sources-value' });
|
||||
|
||||
if (videoData) {
|
||||
release.teaser = [
|
||||
videoData.mp4_large,
|
||||
videoData.webm_large,
|
||||
videoData.mp4,
|
||||
videoData.webm,
|
||||
];
|
||||
}
|
||||
query.url('a[data-videopreview-mp4-large-value]', { attribute: 'data-videopreview-mp4-large-value' }),
|
||||
query.url('a[data-videopreview-mp4-value]', { attribute: 'data-videopreview-mp4-value' }),
|
||||
].filter(Boolean);
|
||||
|
||||
return release;
|
||||
});
|
||||
@@ -150,7 +149,7 @@ async function scrapeScene({ query }, { url, entity }) {
|
||||
}));
|
||||
}
|
||||
|
||||
release.tags = query.contents('div[data-controller] .genres');
|
||||
release.tags = query.contents('a.genres');
|
||||
|
||||
const sourcesData = query.json('.video-container [data-videopreview-sources-value]', { attribute: 'data-videopreview-sources-value' });
|
||||
|
||||
@@ -175,7 +174,7 @@ async function scrapeScene({ query }, { url, entity }) {
|
||||
|| query.attribute('meta[property="og:video"]')
|
||||
|| query.video('video[data-videocontainer-target] source');
|
||||
|
||||
release.photos = query.sourceSets('div[data-controller] a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
|
||||
release.photos = query.sourceSets('a[href^="/photos"] img').map((sourceSet) => sourceSet.map((src) => ({
|
||||
src,
|
||||
expectType: {
|
||||
'application/octet-stream': 'image/jpeg',
|
||||
|
||||
+18
-94
@@ -21,51 +21,33 @@ function scrapeLatest(scenes, site) {
|
||||
|
||||
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeLatestLegacy(scenes, site) {
|
||||
return scenes.map(({ query }) => {
|
||||
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
|
||||
const release = {};
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const domain = new URL(site.url).searchParams.get('d');
|
||||
|
||||
const titleEl = query.element('.galleryTitleText, .articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
|
||||
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
|
||||
if (date) {
|
||||
release.title = title.trim();
|
||||
release.date = date;
|
||||
} else {
|
||||
// title should contain date instead, not applicable in brief mode
|
||||
release.title = title.slice(title.indexOf(':') + 1).trim();
|
||||
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
||||
if (!domain) {
|
||||
return null;
|
||||
}
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
const url = `${site.origin}/is/home.php?o=newest&d=${domain}&p=${page}`;
|
||||
|
||||
const description = query.content('.articleCopyText');
|
||||
if (description) release.description = description.slice(0, description.lastIndexOf('('));
|
||||
|
||||
const duration = query.duration('.articleCopyText a:nth-child(2)');
|
||||
if (duration) release.duration = duration;
|
||||
|
||||
release.likes = query.number('.articlePostDateText td:nth-child(3)');
|
||||
|
||||
const cover = query.img('a img', { origin: site.url });
|
||||
|
||||
release.covers = [[
|
||||
cover.replace('_thumbnail', ''),
|
||||
cover,
|
||||
]];
|
||||
|
||||
return release;
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: '.column.is-3-desktop',
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
// (UNSEXUALLY) BROKEN, abandoned sites, low priority
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, channel, include, cookies) {
|
||||
@@ -97,60 +79,6 @@ async function scrapeScene({ query }, channel, include, cookies) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeSceneLegacy({ query }, { entity: site }) {
|
||||
const release = {};
|
||||
|
||||
const titleEl = query.element('.articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
|
||||
release.title = title.trim();
|
||||
release.description = query.content('.articleCopyText');
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
|
||||
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
|
||||
|
||||
const [cover, ...photos] = query.imgs('img[src*="images"]');
|
||||
release.covers = [cover];
|
||||
release.photos = photos;
|
||||
|
||||
release.poster = query.poster();
|
||||
|
||||
const trailer = query.video('source');
|
||||
if (trailer) release.trailer = { src: trailer };
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: 'body > .columns .column',
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchLatestLegacy(site, page = 1) {
|
||||
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
|
||||
const res = await unprint.get(url); // JSON containing html as a property
|
||||
|
||||
if (res.ok && res.data?.html) {
|
||||
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease, include) {
|
||||
const res = await unprint.get(url, {
|
||||
cookies: { consent: 'yes' },
|
||||
@@ -166,8 +94,4 @@ async function fetchScene(url, site, baseRelease, include) {
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
legacy: {
|
||||
fetchLatest: fetchLatestLegacy,
|
||||
scrapeScene: scrapeSceneLegacy,
|
||||
},
|
||||
};
|
||||
|
||||
+18
-13
@@ -22,12 +22,13 @@ function stripQuery(link) {
|
||||
|
||||
async function getPhotos(albumUrl) {
|
||||
// aggressive Turnstile protection calls for browser
|
||||
const res = await unprint.browser(albumUrl, {
|
||||
const res = await unprint.get(albumUrl, {
|
||||
selectAll: '.photo-thumb',
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return res.context.map(({ query }) => query.sourceSet('source, picture img')).filter(Boolean);
|
||||
return res.context.map(({ query }) => query.img('picture img')).filter(Boolean);
|
||||
}
|
||||
|
||||
return [];
|
||||
@@ -53,14 +54,10 @@ function scrapeAll(scenes, entity) {
|
||||
release.actors = query.content('.models', { trim: false })?.trim().split(/\s{2,}/);
|
||||
}
|
||||
|
||||
const poster = query.sourceSet('img', 'data-srcset')?.[0]
|
||||
// old 640px to 1280px replace won't work anymore with token in URL
|
||||
release.poster = query.sourceSet('img', 'data-srcset')?.[0]
|
||||
|| query.sourceSet('img', 'data-src')?.[0];
|
||||
|
||||
release.poster = poster && [
|
||||
poster.replace('_640', '_1280'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.stars = query.number('.rating');
|
||||
release.likes = query.number('.likes');
|
||||
|
||||
@@ -73,7 +70,8 @@ function scrapeAll(scenes, entity) {
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/video/gallery/${(page - 1) * 12}`; // /0 redirects back to /
|
||||
|
||||
const res = await unprint.browser(url, {
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
selectAll: '.content-grid-item',
|
||||
});
|
||||
|
||||
@@ -88,7 +86,8 @@ async function fetchUpcoming(site) {
|
||||
if (site.parameters?.upcoming) {
|
||||
const url = `${site.url}/video/upcoming`;
|
||||
|
||||
const res = await unprint.browser(url, {
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
selectAll: '.content-grid-item',
|
||||
});
|
||||
|
||||
@@ -140,7 +139,9 @@ async function scrapeScene({ query }, { url, entity, include }) {
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity, _baseRelease, include) {
|
||||
const res = await unprint.browser(url);
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, { url, entity, include });
|
||||
@@ -184,7 +185,9 @@ async function findModel(actor, entity) {
|
||||
|
||||
const url = `${origin}/model/alpha/${firstLetter}`;
|
||||
|
||||
const resModels = await unprint.browser(url);
|
||||
const resModels = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (!resModels.ok) {
|
||||
return resModels.status;
|
||||
@@ -214,7 +217,9 @@ async function fetchProfile(actor, { entity }) {
|
||||
const model = await findModel(actor, entity);
|
||||
|
||||
if (model) {
|
||||
const resModel = await unprint.browser(model.url);
|
||||
const resModel = await unprint.get(model.url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (resModel.ok) {
|
||||
return scrapeProfile(resModel.context, model.avatar);
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const blake2 = require('blake2');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
function getHash(identifier) {
|
||||
const hash = blake2.createHash('blake2b', { digestLength: 8 });
|
||||
|
||||
hash.update(Buffer.from(identifier));
|
||||
|
||||
return hash.digest('hex');
|
||||
return Buffer.from(blake2b(Buffer.from(identifier), { dkLen: 8 })).toString('hex');
|
||||
}
|
||||
|
||||
function extractMaleModelsFromTags(tagContainer) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const mime = require('mime');
|
||||
const mime = require('mime').default;
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const escapeRegexp = require('escape-string-regexp');
|
||||
const escapeRegexp = require('escape-string-regexp').default;
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const capitalize = require('../utils/capitalize');
|
||||
|
||||
+14
-29
@@ -47,11 +47,7 @@ function scrapeAll(scenes, tilesByEntryId, channel) {
|
||||
|
||||
release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
|
||||
if (posterBackground?.background) {
|
||||
release.poster = posterBackground.background.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
}
|
||||
release.poster = query.background('.player');
|
||||
|
||||
release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
|
||||
unprint.query.url(linkEl, null),
|
||||
@@ -121,8 +117,7 @@ function scrapeAllTiles(tiles, channel) {
|
||||
|
||||
release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.scene-img');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.scene-img');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -199,8 +194,7 @@ function scrapeAllCospuri(scenes, channel) {
|
||||
|
||||
release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.scene-thumb');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.scene-thumb');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -260,8 +254,7 @@ function scrapeAllFellatio(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('.tour-data a[href*="tag/"]');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -302,8 +295,7 @@ function scrapeAllHandjob(scenes, _channel) {
|
||||
|
||||
release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -348,8 +340,7 @@ function scrapeAllLegs(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('a[href*="tag/"]');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -385,8 +376,7 @@ function scrapeAllFacefuck(scenes) {
|
||||
release.description = query.content('.infotxt');
|
||||
release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -425,8 +415,7 @@ function scrapeAllTrans(scenes) {
|
||||
release.duration = query.duration('.sample-info');
|
||||
release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -436,7 +425,7 @@ function scrapeAllTrans(scenes) {
|
||||
];
|
||||
}
|
||||
|
||||
release.photos = curatePhotos(albumContext?.query.styles('.sample-lg, .sample-thumb').map((style) => style['background-image']?.match(/url\((.*)\)/)?.[1]));
|
||||
release.photos = curatePhotos(albumContext?.query.backgrounds('.sample-lg, .sample-thumb'));
|
||||
release.trailer = query.video();
|
||||
|
||||
release.entryId = entryIdFromMedia(release);
|
||||
@@ -486,8 +475,7 @@ function scrapeAllLesbianTiles(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('.content-tags a');
|
||||
|
||||
const posterBackground = query.style('.vidthumb');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.vidthumb');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -522,8 +510,7 @@ function scrapeAllLesbian(scenes, channel, tiles) {
|
||||
release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
|
||||
release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -650,8 +637,7 @@ function scrapeSceneBuffet({ query }, { url, entity }) {
|
||||
|
||||
release.tags = query.contents('.tag-list a');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -696,8 +682,7 @@ function scrapeSceneCospuri({ query }, { url, entity }) {
|
||||
|
||||
release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -788,7 +773,7 @@ function scrapeProfile({ query }, _channel, url) {
|
||||
|
||||
const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
|
||||
|| query.img('.costume-bg', { attribute: 'data-img' })
|
||||
|| query.style('.model-profile, #profile, .carousel-item')?.['background-image']?.match(/url\((.*)\)/)?.[1];
|
||||
|| query.background('.model-profile, #profile, .carousel-item');
|
||||
|
||||
if (avatar) {
|
||||
profile.avatar = Array.from(new Set([
|
||||
|
||||
@@ -47,7 +47,7 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
|
||||
// release.entryId = scene.id; // legacy
|
||||
release.entryId = scene.itemId;
|
||||
|
||||
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
|
||||
release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
|
||||
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
@@ -58,7 +58,7 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
|
||||
name: model.modelName || model.name || model.title,
|
||||
avatar: model.img,
|
||||
gender: model.gender,
|
||||
url: `${channel.url}/models/${model.modelId || model.id}`,
|
||||
url: `${channel.origin}/models/${model.modelId || model.id}`,
|
||||
}));
|
||||
|
||||
if (scene.img) {
|
||||
@@ -94,7 +94,7 @@ function scrapeAll(scenes, channel, parameters) {
|
||||
|
||||
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
// url: 'https://www.pervz.com/series/pervz-features',
|
||||
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
|
||||
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
|
||||
|
||||
const url = seriesId
|
||||
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
|
||||
|
||||
@@ -241,8 +241,8 @@ function generateActors(release) {
|
||||
: Math.floor(Math.random() * 3) + 2;
|
||||
|
||||
return Array.from({ length }, () => ({
|
||||
name: faker.name
|
||||
.findName()
|
||||
name: faker.person
|
||||
.fullName()
|
||||
.split(' ')
|
||||
.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
|
||||
.join(' '),
|
||||
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
|
||||
const knex = require('./knex');
|
||||
const { indexApi } = require('./manticore');
|
||||
const { HttpError } = require('./errors');
|
||||
const slugify = require('./utils/slugify');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
let lastActorsViewRefresh = 0;
|
||||
|
||||
function curateStash(stash) {
|
||||
if (!stash) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const curatedStash = {
|
||||
id: stash.id,
|
||||
name: stash.name,
|
||||
slug: stash.slug,
|
||||
primary: stash.primary,
|
||||
};
|
||||
|
||||
return curatedStash;
|
||||
}
|
||||
|
||||
function curateStashEntry(stash, user) {
|
||||
const curatedStashEntry = {
|
||||
user_id: user.id,
|
||||
name: stash.name,
|
||||
slug: slugify(stash.name),
|
||||
public: false,
|
||||
};
|
||||
|
||||
return curatedStashEntry;
|
||||
}
|
||||
|
||||
async function fetchStash(stashId, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const stash = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.first();
|
||||
|
||||
if (!stash) {
|
||||
throw new HttpError('You are not authorized to access this stash', 403);
|
||||
}
|
||||
|
||||
return curateStash(stash);
|
||||
}
|
||||
|
||||
async function fetchStashes(domain, itemId, sessionUser) {
|
||||
const stashes = await knex(`stashes_${domain}s`)
|
||||
.select('stashes.*')
|
||||
.where({
|
||||
[`${domain}_id`]: itemId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
||||
|
||||
return stashes.map((stash) => curateStash(stash));
|
||||
}
|
||||
|
||||
async function createStash(newStash, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
try {
|
||||
const stash = await knex('stashes')
|
||||
.insert(curateStashEntry(newStash, sessionUser))
|
||||
.returning('*');
|
||||
|
||||
return curateStash(stash);
|
||||
} catch (error) {
|
||||
if (error.routine === '_bt_check_unique') {
|
||||
throw new HttpError('Stash name should be unique', 409);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateStash(stashId, newStash, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const stash = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.update(newStash)
|
||||
.returning('*');
|
||||
|
||||
if (!stash) {
|
||||
throw new HttpError('You are not authorized to modify this stash', 403);
|
||||
}
|
||||
|
||||
return curateStash(stash);
|
||||
}
|
||||
|
||||
async function removeStash(stashId, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const removed = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
primary: false,
|
||||
})
|
||||
.delete();
|
||||
|
||||
if (removed === 0) {
|
||||
throw new HttpError('Unable to remove this stash', 400);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshActorsView() {
|
||||
if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
|
||||
// don't refresh actors view more than once an hour
|
||||
lastActorsViewRefresh = new Date();
|
||||
|
||||
logger.debug('Refreshing actors view');
|
||||
|
||||
return knex.schema.refreshMaterializedView('actors_meta');
|
||||
}
|
||||
|
||||
logger.silly('Skipping actors view refresh');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function stashActor(actorId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_actors')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
actor_id: actorId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'actors_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
actor_id: actorId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
refreshActorsView();
|
||||
|
||||
return fetchStashes('actor', actorId, sessionUser);
|
||||
}
|
||||
|
||||
async function stashScene(sceneId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_scenes')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
scene_id: sceneId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'scenes_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
// ...doc.replace.doc,
|
||||
scene_id: sceneId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('scene', sceneId, sessionUser);
|
||||
}
|
||||
|
||||
async function stashMovie(movieId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_movies')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
movie_id: movieId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'movies_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
movie_id: movieId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('movie', movieId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashActor(actorId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_actors AS deletable')
|
||||
.where('deletable.actor_id', actorId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_actors') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
|
||||
.where('stashes_actors.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'actors_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { actor_id: actorId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
refreshActorsView();
|
||||
|
||||
return fetchStashes('actor', actorId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashScene(sceneId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_scenes AS deletable')
|
||||
.where('deletable.scene_id', sceneId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_scenes') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
||||
.where('stashes_scenes.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'scenes_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { scene_id: sceneId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('scene', sceneId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashMovie(movieId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_movies AS deletable')
|
||||
.where('deletable.movie_id', movieId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_movies') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id')
|
||||
.where('stashes_movies.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'movies_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { movie_id: movieId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('movie', movieId, sessionUser);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createStash,
|
||||
curateStash,
|
||||
removeStash,
|
||||
stashActor,
|
||||
stashScene,
|
||||
stashMovie,
|
||||
unstashScene,
|
||||
unstashActor,
|
||||
unstashMovie,
|
||||
updateStash,
|
||||
};
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const { decode } = require('html-entities');
|
||||
const { decode: decodeEntities } = require('html-entities');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const argv = require('./argv');
|
||||
@@ -28,6 +28,11 @@ const commonContext = {
|
||||
unprint,
|
||||
};
|
||||
|
||||
// newer html-entities throws on non-strings, pass e.g. numeric titles through like 2.4 did
|
||||
function decode(text) {
|
||||
return typeof text === 'string' ? decodeEntities(text) : text;
|
||||
}
|
||||
|
||||
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
||||
const slugBase = release.title
|
||||
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const bhttp = require('bhttp');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { studios: oldStudios } = require('../../seeds/03_studios');
|
||||
|
||||
/*
|
||||
const slugMap = {
|
||||
thay_ksada: 'thayksada2',
|
||||
};
|
||||
*/
|
||||
|
||||
async function init() {
|
||||
const res = await bhttp.get('https://pornbox.com/studio/list');
|
||||
|
||||
if (!res.statusCode === 200) {
|
||||
throw new Error(`Failed to retrieve studios (${res.statusCode}): ${res.body.toString()}`);
|
||||
}
|
||||
|
||||
const newStudios = res.body.map((studio) => {
|
||||
const slug = slugify(studio.name, '');
|
||||
const oldStudio = oldStudios.find((oldStudioX) => oldStudioX.slug === slug);
|
||||
|
||||
const tags = oldStudio?.tags || [];
|
||||
|
||||
const newStudio = {
|
||||
name: studio.name,
|
||||
slug,
|
||||
originalSlug: slugify(studio.alias, ''),
|
||||
url: `https://www.analvids.com/studios/${studio.alias}`,
|
||||
parent: 'analvids',
|
||||
};
|
||||
|
||||
if (oldStudio?.alias) {
|
||||
newStudio.alias = oldStudio.alias;
|
||||
}
|
||||
|
||||
if (studio.description && studio.description.toLowerCase() !== studio.name.toLowerCase()) {
|
||||
newStudio.description = studio.description;
|
||||
}
|
||||
|
||||
if (studio.directions?.includes('gay')) {
|
||||
tags.push('gay');
|
||||
}
|
||||
|
||||
if (studio.directions?.includes('lesbian')) {
|
||||
tags.push('lesbian');
|
||||
}
|
||||
|
||||
if (studio.directions?.includes('trans')) {
|
||||
tags.push('transsexual');
|
||||
}
|
||||
|
||||
if (tags.length > 0) {
|
||||
newStudio.tags = Array.from(new Set(tags));
|
||||
}
|
||||
|
||||
return newStudio;
|
||||
});
|
||||
|
||||
await fs.writeFile('./analvids-studios.json', JSON.stringify(newStudios, null, 4));
|
||||
|
||||
newStudios.reduce((acc, studio) => {
|
||||
const existingStudio = acc.get(studio.slug);
|
||||
|
||||
if (existingStudio) {
|
||||
if (!acc.has(studio.originalSlug)) {
|
||||
console.log('REWRITE', studio.slug, studio.originalSlug);
|
||||
studio.slug = studio.originalSlug; // eslint-disable-line
|
||||
}
|
||||
|
||||
console.log('slug already exists!', existingStudio, studio, studio.originalSlug, studio.slug);
|
||||
}
|
||||
|
||||
acc.set(studio.slug, studio);
|
||||
|
||||
return acc;
|
||||
}, new Map());
|
||||
|
||||
// console.log(newStudios);
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,18 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
async function init() {
|
||||
const res = await bhttp.head('https://www.kink.com/consent', {
|
||||
headers: {
|
||||
Host: 'www.kink.com',
|
||||
'User-Agent': 'HTTPie/2.6.0',
|
||||
Accept: '*/*',
|
||||
Connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
|
||||
console.log(res.body.toString(), res.headers, res.statusCode);
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,40 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
const express = require('express');
|
||||
|
||||
async function init() {
|
||||
const res = await fetch('https://www.realitykings.com/scenes?site=45', {
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
'user-agent': 'HTTPie/2.6.0',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
accept: '*/*',
|
||||
connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
|
||||
console.log(res.status, res.headers);
|
||||
|
||||
const app = express();
|
||||
|
||||
app.get('/', (appReq, appRes) => {
|
||||
console.log(appReq.headers);
|
||||
appRes.status(204).send();
|
||||
});
|
||||
|
||||
app.listen(8000, () => {
|
||||
console.log('Listening on port 8000');
|
||||
|
||||
fetch('http://127.0.0.1:8000', {
|
||||
headers: {
|
||||
'user-agent': 'HTTPie/2.6.0',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
accept: '*/*',
|
||||
connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,17 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const express = require('express');
|
||||
|
||||
const app = express();
|
||||
|
||||
app.get('/', (req, res) => {
|
||||
console.log(req.httpVersion);
|
||||
console.log(req.headers);
|
||||
console.log(req.body);
|
||||
|
||||
res.status(204).send();
|
||||
});
|
||||
|
||||
app.listen(5555, () => {
|
||||
console.log('Server started');
|
||||
});
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
// const util = require('util');
|
||||
const unprint = require('unprint');
|
||||
const args = require('yargs').argv;
|
||||
const args = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
|
||||
|
||||
const knex = require('../knex');
|
||||
const bulkInsert = require('../utils/bulk-insert');
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('./knex');
|
||||
const { curateStash } = require('./stashes');
|
||||
|
||||
function curateUser(user) {
|
||||
if (!user) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ability = [...(user.role_abilities || []), ...(user.abilities || [])];
|
||||
|
||||
const curatedUser = {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
email: user.email,
|
||||
emailVerified: user.email_verified,
|
||||
identityVerified: user.identity_verified,
|
||||
ability,
|
||||
createdAt: user.created_at,
|
||||
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
|
||||
};
|
||||
|
||||
return curatedUser;
|
||||
}
|
||||
|
||||
async function fetchUser(userId, raw) {
|
||||
const user = await knex('users')
|
||||
.select(knex.raw('users.*, users_roles.abilities as role_abilities, COALESCE(json_agg(stashes ORDER BY stashes.created_at) FILTER (WHERE stashes.id IS NOT NULL), \'[]\') as stashes'))
|
||||
.modify((builder) => {
|
||||
if (typeof userId === 'number') {
|
||||
builder.where('users.id', userId);
|
||||
}
|
||||
|
||||
if (typeof userId === 'string') {
|
||||
builder
|
||||
.where('users.username', userId)
|
||||
.orWhere('users.email', userId);
|
||||
}
|
||||
})
|
||||
.leftJoin('users_roles', 'users_roles.role', 'users.role')
|
||||
.leftJoin('stashes', 'stashes.user_id', 'users.id')
|
||||
.groupBy('users.id', 'users_roles.role')
|
||||
.first();
|
||||
|
||||
if (raw) {
|
||||
return user;
|
||||
}
|
||||
|
||||
return curateUser(user);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
curateUser,
|
||||
fetchUser,
|
||||
};
|
||||
@@ -1,118 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const Promise = require('bluebird');
|
||||
const fsPromises = require('fs').promises;
|
||||
const fs = require('fs');
|
||||
const { PassThrough } = require('stream');
|
||||
const blake2 = require('blake2');
|
||||
const argv = require('yargs').argv;
|
||||
|
||||
const file = 'https://speed.hetzner.de/100MB.bin';
|
||||
// const file = 'https://speed.hetzner.de/1GB.bin';
|
||||
// const file = 'https://speed.hetzner.de/10GB.bin';
|
||||
|
||||
function getMemoryUsage() {
|
||||
return process.memoryUsage().rss / (10 ** 6);
|
||||
}
|
||||
|
||||
const stats = {
|
||||
peakMemoryUsage: getMemoryUsage(),
|
||||
done: false,
|
||||
downloads: {},
|
||||
};
|
||||
|
||||
function render() {
|
||||
const downloads = Object.entries(stats.downloads);
|
||||
|
||||
process.stdout.clearScreenDown();
|
||||
|
||||
process.stdout.write(`peak memory: ${stats.peakMemoryUsage.toFixed(2)} MB\n`);
|
||||
|
||||
downloads.forEach(([download, progress]) => {
|
||||
process.stdout.write(`${download}: ${progress}${typeof progress === 'string' ? '' : '%'}\n`);
|
||||
});
|
||||
|
||||
process.stdout.moveCursor(0, -(downloads.length + 1));
|
||||
process.stdout.cursorTo(0);
|
||||
|
||||
if (downloads.length === 0 || !downloads.every(([_label, download]) => typeof download === 'string')) {
|
||||
setTimeout(() => render(), 1000);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.moveCursor(0, downloads.length + 1);
|
||||
}
|
||||
|
||||
function setProgress(label, completedBytes, totalBytes, hash) {
|
||||
const memory = getMemoryUsage();
|
||||
|
||||
stats.peakMemoryUsage = Math.max(memory, stats.peakMemoryUsage);
|
||||
stats.downloads[label] = hash || Math.round((completedBytes / totalBytes) * 100);
|
||||
}
|
||||
|
||||
async function buffered(label) {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
|
||||
const imageRes = await bhttp.get(file, {
|
||||
onDownloadProgress(completedBytes, totalBytes) {
|
||||
setProgress(label, completedBytes, totalBytes);
|
||||
},
|
||||
});
|
||||
|
||||
hash.update(imageRes.body);
|
||||
setProgress(label, null, null, hash.digest('hex'));
|
||||
|
||||
await fsPromises.writeFile(`/mnt/stor/Pictures/traxxx/temp/buffered-${label}.bin`, imageRes.body);
|
||||
}
|
||||
|
||||
async function streamed(label) {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
hash.setEncoding('hex');
|
||||
|
||||
const hashStream = new PassThrough();
|
||||
const targetStream = fs.createWriteStream(`/mnt/stor/Pictures/traxxx/temp/streamed-${label}.bin`);
|
||||
|
||||
const imageRes = await bhttp.get(file, {
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const stream = imageRes
|
||||
.pipe(hashStream)
|
||||
.pipe(targetStream);
|
||||
|
||||
imageRes.on('progress', (completedBytes, totalBytes) => {
|
||||
setProgress(label, completedBytes, totalBytes);
|
||||
});
|
||||
|
||||
hashStream.on('data', (chunk) => {
|
||||
hash.write(chunk);
|
||||
});
|
||||
|
||||
stream.on('finish', () => {
|
||||
hash.end();
|
||||
setProgress(label, null, null, hash.read());
|
||||
});
|
||||
}
|
||||
|
||||
async function init() {
|
||||
const n = argv.n || 1;
|
||||
|
||||
if (argv._.includes('stream')) {
|
||||
console.log('using streams');
|
||||
render();
|
||||
|
||||
await Promise.map(Array.from({ length: n }), async (value, index) => streamed(index + 1));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (argv._.includes('buffer')) {
|
||||
console.log('using buffers');
|
||||
render();
|
||||
|
||||
await Promise.map(Array.from({ length: n }), async (value, index) => buffered(index + 1));
|
||||
}
|
||||
}
|
||||
|
||||
init();
|
||||
-153
@@ -1,153 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const arrayEqual = require('array-equal');
|
||||
|
||||
/* eslint-disable */
|
||||
function evaluateBranch(tree, modifiers) {
|
||||
const result = tree.map((expression) => {
|
||||
if (expression.type === 'group') {
|
||||
return evaluateBranch(expression.values, expression.modifiers);
|
||||
} if (expression.modifiers.length === 0) {
|
||||
return ''; // This is a trigger to stringify the previous values
|
||||
} if (arrayEqual(['plus'], expression.modifiers)) {
|
||||
return 0;
|
||||
} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
|
||||
return 1;
|
||||
}
|
||||
throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
|
||||
}).reduce((combined, value) => {
|
||||
if (value === '') {
|
||||
return combined.toString();
|
||||
}
|
||||
if (value === true) {
|
||||
value = 1;
|
||||
}
|
||||
|
||||
if (typeof combined === 'string') {
|
||||
return combined + value.toString();
|
||||
}
|
||||
return combined + value;
|
||||
}, 0);
|
||||
|
||||
if (modifiers == null) {
|
||||
return result;
|
||||
}
|
||||
if (arrayEqual(['plus'], modifiers)) {
|
||||
return parseInt(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function evaluate(tree) {
|
||||
return evaluateBranch(tree);
|
||||
}
|
||||
|
||||
function parse(string) {
|
||||
const length = string.length;
|
||||
let byte;
|
||||
|
||||
let stateFinishedItem = false;
|
||||
const modifierStack = [[]];
|
||||
const itemStack = [[]];
|
||||
let currentStack = itemStack[0];
|
||||
let currentModifiers = modifierStack[0];
|
||||
let stackLevel = 0;
|
||||
|
||||
for (let pos = 0; pos < length; pos++) {
|
||||
byte = string[pos];
|
||||
|
||||
switch (byte) {
|
||||
case '+':
|
||||
if (pos === 0 || stateFinishedItem === false) {
|
||||
// Modifier / number-cast
|
||||
currentModifiers.push('plus');
|
||||
stateFinishedItem = false;
|
||||
} else {
|
||||
// Addition, we don't need to do anything here
|
||||
}
|
||||
break;
|
||||
case '!':
|
||||
stateFinishedItem = false;
|
||||
currentModifiers.push('negate');
|
||||
break;
|
||||
case '(':
|
||||
stateFinishedItem = false;
|
||||
stackLevel++;
|
||||
itemStack[stackLevel] = currentStack = [];
|
||||
modifierStack[stackLevel] = currentModifiers = [];
|
||||
break;
|
||||
case ')':
|
||||
if (stackLevel === 0) {
|
||||
throw new Error('Encountered ) without matching (');
|
||||
}
|
||||
|
||||
stackLevel--;
|
||||
stateFinishedItem = true;
|
||||
|
||||
currentStack = itemStack[stackLevel];
|
||||
currentStack.push({
|
||||
type: 'group',
|
||||
values: itemStack[stackLevel + 1],
|
||||
modifiers: modifierStack[stackLevel],
|
||||
});
|
||||
|
||||
currentModifiers = modifierStack[stackLevel] = [];
|
||||
break;
|
||||
case '[':
|
||||
if (string[pos + 1] === ']') {
|
||||
// Reached the brackets; end of the modifier sequence
|
||||
currentStack.push({
|
||||
type: 'brackets',
|
||||
modifiers: currentModifiers,
|
||||
});
|
||||
|
||||
currentModifiers = [];
|
||||
pos += 1; // Skip over the closing bracket
|
||||
stateFinishedItem = true;
|
||||
} else {
|
||||
throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return itemStack[0];
|
||||
}
|
||||
|
||||
function parseExpression(string) {
|
||||
return evaluate(parse(string));
|
||||
}
|
||||
|
||||
function findAll(regex, target) {
|
||||
const results = []; let
|
||||
match;
|
||||
|
||||
while (match = regex.exec(target)) {
|
||||
results.push(match);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function cf(testcase) {
|
||||
let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
|
||||
|
||||
const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
|
||||
operation: match[1],
|
||||
expression: match[2],
|
||||
})).map(({ operation, expression }) => ({
|
||||
operation,
|
||||
expression: parseExpression(expression),
|
||||
}));
|
||||
|
||||
initialExpression = parseExpression(initialExpression);
|
||||
|
||||
return { parent, child, initialExpression, modifyingExpressions };
|
||||
};
|
||||
|
||||
module.exports = cf;
|
||||
@@ -82,7 +82,7 @@ function convertApi(rawInput, fromOrTo, to) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const input = decode(rawInput); // remove html entities, e.g. 5' 8" for 5' 8"
|
||||
const input = typeof rawInput === 'string' ? decode(rawInput) : rawInput; // remove html entities, e.g. 5' 8" for 5' 8"
|
||||
|
||||
try {
|
||||
if (typeof input === 'string' && to === undefined) {
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = new Map();
|
||||
@@ -1,511 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const undici = require('undici');
|
||||
const fs = require('fs').promises;
|
||||
// const util = require('util');
|
||||
// const stream = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
const Bottleneck = require('bottleneck');
|
||||
const { JSDOM, toughCookie } = require('jsdom');
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
|
||||
const windows = require('./http-windows');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
const virtualConsole = require('./virtual-console')(__filename);
|
||||
const argv = require('../argv');
|
||||
|
||||
// const pipeline = util.promisify(stream.pipeline);
|
||||
|
||||
const limiters = {
|
||||
bypass: new Bottleneck({
|
||||
minTime: 1000,
|
||||
maxConcurrent: 1,
|
||||
timeout: 60000,
|
||||
}),
|
||||
};
|
||||
|
||||
const bypassSessions = new Map();
|
||||
|
||||
const browsers = new Map();
|
||||
|
||||
Promise.config({
|
||||
cancellation: true,
|
||||
});
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
const defaultOptions = {
|
||||
timeout: argv.requestTimeout,
|
||||
encodeJSON: true,
|
||||
parse: false,
|
||||
headers: {
|
||||
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
|
||||
},
|
||||
};
|
||||
|
||||
function useProxy(url) {
|
||||
if (!config.proxy.enable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
return config.proxy.hostnames.includes(hostname);
|
||||
}
|
||||
|
||||
function useBrowserBypass(url, options) {
|
||||
if (!config.bypass.browser.enable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
if (options.bypassBrowser === 'shared') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (config.bypass.browser.hostnames.includes(hostname)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function useCloudflareBypass(url, options) {
|
||||
if (!config.bypass.cloudflare.enable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
if (options.bypassCloudflare === 'shared') {
|
||||
return 'shared';
|
||||
}
|
||||
|
||||
if (options.bypassCloudflare === 'independent') {
|
||||
return hostname;
|
||||
}
|
||||
|
||||
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
|
||||
return 'shared';
|
||||
}
|
||||
|
||||
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
|
||||
return hostname;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const propMap = {
|
||||
media: {
|
||||
interval: 'mediaInterval',
|
||||
concurrency: 'mediaConcurrency',
|
||||
},
|
||||
};
|
||||
|
||||
function getLimiterValue(prop, options, hostname) {
|
||||
const mappedProp = propMap[options.limits]?.[prop] || prop;
|
||||
|
||||
if (typeof argv[mappedProp] !== 'undefined') {
|
||||
return argv[mappedProp];
|
||||
}
|
||||
|
||||
if (typeof options[prop] !== 'undefined') {
|
||||
return options[prop];
|
||||
}
|
||||
|
||||
if (config.limits[hostname]?.enable !== false && typeof config.limits[hostname]?.[prop] !== 'undefined') {
|
||||
return config.limits[hostname][prop];
|
||||
}
|
||||
|
||||
return config.limits[options.limits || 'default'][prop];
|
||||
}
|
||||
|
||||
function getLimiter(options = {}, url) {
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
const interval = getLimiterValue('interval', options, hostname);
|
||||
const concurrency = getLimiterValue('concurrency', options, hostname);
|
||||
|
||||
if (!limiters[interval]?.[concurrency]) {
|
||||
limiters[interval] = limiters[interval] || {};
|
||||
|
||||
limiters[interval][concurrency] = new Bottleneck({
|
||||
minTime: interval,
|
||||
maxConcurrent: concurrency,
|
||||
timeout: (options.timeout || defaultOptions.timeout) + 10000, // timeout 10 seconds after bhttp should
|
||||
});
|
||||
}
|
||||
|
||||
return limiters[interval][concurrency];
|
||||
}
|
||||
|
||||
function extractJson(body, headers) {
|
||||
if (headers['content-type'].includes('application/json')) {
|
||||
const { document } = new JSDOM(body, { virtualConsole }).window;
|
||||
const dataString = document.querySelector('body > pre')?.textContent;
|
||||
|
||||
if (dataString) {
|
||||
const data = JSON.parse(dataString);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
async function getBrowserSession(identifier, options = {}) {
|
||||
return limiters.bypass.schedule(async () => {
|
||||
const browserIdentifier = options.useGlobalBrowser === false ? identifier : null;
|
||||
|
||||
if (!browsers.has(browserIdentifier)) {
|
||||
const newBrowser = await puppeteer.launch({
|
||||
headless: typeof options.headless === 'undefined' ? 'new' : options.headless,
|
||||
args: [
|
||||
...(options.useProxy && config.proxy.enabled ? [`--proxy-server=${config.proxy.host}:${config.proxy.port}`] : []),
|
||||
],
|
||||
// headless: false,
|
||||
});
|
||||
|
||||
browsers.set(browserIdentifier, newBrowser);
|
||||
|
||||
logger.info('Initialized puppeteer browser');
|
||||
}
|
||||
|
||||
const browser = browsers.get(browserIdentifier);
|
||||
const tab = await browser.newPage();
|
||||
|
||||
logger.verbose(`Opened puppeteer tab${identifier ? ` for ${identifier}` : ''}`);
|
||||
|
||||
return { browser, tab };
|
||||
});
|
||||
}
|
||||
|
||||
async function bypassBrowserRequest(url, options) {
|
||||
const { tab } = await getBrowserSession(new URL(url).hostname, options);
|
||||
|
||||
const res = await tab.goto(url);
|
||||
|
||||
if (options.bypass?.delay) {
|
||||
await Promise.delay(options.bypass.delay);
|
||||
}
|
||||
|
||||
if (typeof options.bypass?.evaluate === 'function') {
|
||||
await tab.evaluate(options.bypass.evaluate, options.bypass);
|
||||
}
|
||||
|
||||
const rawBody = await tab.content();
|
||||
|
||||
const headers = res.headers();
|
||||
const body = extractJson(rawBody, headers);
|
||||
|
||||
const statusCode = res.status();
|
||||
|
||||
if (!statusCode === 200) {
|
||||
throw new Error(`Puppeteer bypass failed for ${url} (${statusCode}): ${body?.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
body,
|
||||
statusCode,
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
async function getBypassSession(url, hostname) {
|
||||
if (bypassSessions.has(hostname)) {
|
||||
return bypassSessions.get(hostname);
|
||||
}
|
||||
|
||||
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.create',
|
||||
proxy: useProxy(url) ? {
|
||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
||||
} : null,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
|
||||
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
|
||||
}
|
||||
|
||||
bypassSessions.set(hostname, sessionRes.body.session);
|
||||
|
||||
return sessionRes.body.session;
|
||||
}
|
||||
|
||||
async function destroyBypassSession(sessionId) {
|
||||
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.destroy',
|
||||
session: sessionId,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
|
||||
bypassSessions.delete(sessionId);
|
||||
|
||||
logger.verbose(`Destroyed bypass session ${sessionId}`);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function destroyBypassSessions() {
|
||||
if (!config.bypass.cloudflare.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.list',
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
|
||||
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
|
||||
}
|
||||
|
||||
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
|
||||
}
|
||||
|
||||
async function destroyBrowserSessions() {
|
||||
await Promise.all(Array.from(browsers.values(), async (browser) => browser?.close()));
|
||||
}
|
||||
|
||||
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
|
||||
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
|
||||
|
||||
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
|
||||
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: `request.${method}`,
|
||||
url,
|
||||
session: sessionId,
|
||||
maxTimeout: options.timeout,
|
||||
proxy: useProxy(url) ? {
|
||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
||||
} : null,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
|
||||
if (/session closed/i.test(res.body?.message) && attempts < 3) {
|
||||
await destroyBypassSession(sessionId);
|
||||
|
||||
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
|
||||
}
|
||||
|
||||
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
|
||||
}
|
||||
|
||||
const resBody = extractJson(res.body.solution.response, res.body.solution.headers);
|
||||
|
||||
return {
|
||||
body: resBody,
|
||||
statusCode: res.body.solution.status,
|
||||
headers: res.body.solution.headers,
|
||||
};
|
||||
}
|
||||
|
||||
const defaultAgent = new undici.Agent({
|
||||
allowH2: true,
|
||||
connect: {
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
});
|
||||
|
||||
const proxyAgent = new undici.ProxyAgent({
|
||||
uri: `http://${config.proxy.host}:${config.proxy.port}`,
|
||||
});
|
||||
|
||||
async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
|
||||
const withProxy = useProxy(url);
|
||||
const withBrowserBypass = useBrowserBypass(url, requestOptions);
|
||||
const withCloudflareBypass = useCloudflareBypass(url, requestOptions);
|
||||
|
||||
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
|
||||
|
||||
if (withBrowserBypass) {
|
||||
if (method !== 'get') throw new Error('Browser bypass only supports GET');
|
||||
return bypassBrowserRequest(url, requestOptions);
|
||||
}
|
||||
|
||||
if (withCloudflareBypass) {
|
||||
return bypassCloudflareRequest(url, method, body, withCloudflareBypass, requestOptions);
|
||||
}
|
||||
|
||||
const headers = {
|
||||
...requestOptions.headers,
|
||||
};
|
||||
|
||||
const res = await undici.request(url, {
|
||||
method: method.toUpperCase(),
|
||||
headers,
|
||||
body: body ?? null,
|
||||
dispatcher: withProxy
|
||||
? proxyAgent
|
||||
: defaultAgent,
|
||||
maxRedirections: 0, // handle manually
|
||||
});
|
||||
|
||||
if (res.headers.location && redirects < 3) {
|
||||
// Drain the body to free the socket before redirecting
|
||||
await res.body.dump();
|
||||
const nextUrl = new URL(res.headers.location, url).href;
|
||||
return request(method, nextUrl, body, requestOptions, limiter, redirects + 1);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async function finalizeResult(res, url, options) {
|
||||
if (options.destination) {
|
||||
await pipeline(
|
||||
res.body,
|
||||
...(options.transforms || []),
|
||||
options.destination,
|
||||
);
|
||||
|
||||
return {
|
||||
statusCode: res.statusCode,
|
||||
headers: res.headers,
|
||||
status: res.statusCode,
|
||||
ok: res.statusCode >= 200 && res.statusCode <= 299,
|
||||
};
|
||||
}
|
||||
|
||||
const buffer = await res.body.arrayBuffer();
|
||||
const html = Buffer.from(buffer).toString();
|
||||
const window = options?.parse ? new JSDOM(html, { virtualConsole, ...options.extract }).window : null;
|
||||
const pathname = new URL(url).pathname.replace(/\//g, '_');
|
||||
|
||||
if (window && /fetchScene|fetchMovie/.test(new Error().stack)) {
|
||||
windows.set(pathname, window);
|
||||
}
|
||||
|
||||
if (argv.saveHtml) {
|
||||
await fs.writeFile(`./html/${pathname}.html`, html);
|
||||
}
|
||||
|
||||
return {
|
||||
statusCode: res.statusCode,
|
||||
status: res.statusCode,
|
||||
headers: res.headers,
|
||||
body: html,
|
||||
html,
|
||||
document: window?.document || null,
|
||||
window,
|
||||
ok: res.statusCode >= 200 && res.statusCode <= 299,
|
||||
};
|
||||
}
|
||||
|
||||
function getTimeout(options, url) {
|
||||
return new Promise((_resolve, reject, onCancel) => {
|
||||
const timeout = setTimeout(() => {
|
||||
logger.debug(`Canceled timed out request to ${url}`);
|
||||
reject(new Error(`URL ${url} timed out`));
|
||||
}, (options?.timeout || defaultOptions.timeout) + 10000);
|
||||
|
||||
onCancel(() => {
|
||||
clearTimeout(timeout);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
|
||||
if (typeof url !== 'string') {
|
||||
console.trace(`Bad URL: ${JSON.stringify(url)}`);
|
||||
}
|
||||
|
||||
const options = {
|
||||
...defaultOptions,
|
||||
...requestOptions,
|
||||
headers: {
|
||||
...Object.fromEntries(requestOptions.includeDefaultHeaders === false ? {} : Object.entries(defaultOptions.headers).map(([key, value]) => [key.toLowerCase(), value])),
|
||||
...Object.fromEntries(Object.entries(requestOptions.headers || {}).map(([key, value]) => [key.toLowerCase(), value])),
|
||||
},
|
||||
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
|
||||
stream: !!requestOptions.destination,
|
||||
};
|
||||
|
||||
const limiter = getLimiter(options, url);
|
||||
const timeout = getTimeout(options, url);
|
||||
|
||||
const result = await limiter.schedule(async () => Promise.race([request(method, url, body, options, limiter), timeout]));
|
||||
|
||||
timeout.cancel();
|
||||
|
||||
const curatedResult = await finalizeResult(result, url, options);
|
||||
|
||||
logger.silly(`Response ${curatedResult.status} for ${method.toUpperCase()} ${url}`);
|
||||
|
||||
return curatedResult;
|
||||
}
|
||||
|
||||
async function get(url, options) {
|
||||
return scheduleRequest('get', url, null, options);
|
||||
}
|
||||
|
||||
async function post(url, body, options) {
|
||||
return scheduleRequest('post', url, body, options);
|
||||
}
|
||||
|
||||
async function put(url, body, options) {
|
||||
return scheduleRequest('put', url, body, options);
|
||||
}
|
||||
|
||||
async function patch(url, body, options) {
|
||||
return scheduleRequest('patch', url, body, options);
|
||||
}
|
||||
|
||||
async function del(url, options) {
|
||||
return scheduleRequest('delete', url, null, options);
|
||||
}
|
||||
|
||||
async function head(url, options) {
|
||||
return scheduleRequest('head', url, null, options);
|
||||
}
|
||||
|
||||
function getSession(options) {
|
||||
return bhttp.session({ ...defaultOptions, ...options });
|
||||
}
|
||||
|
||||
function getCookieJar(store, options) {
|
||||
return new toughCookie.CookieJar(store, {
|
||||
looseMode: true,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
toughCookie,
|
||||
get,
|
||||
head,
|
||||
post,
|
||||
delete: del,
|
||||
put,
|
||||
patch,
|
||||
session: getSession,
|
||||
cookieJar: getCookieJar,
|
||||
browsers,
|
||||
getBrowserSession,
|
||||
getBypassSession,
|
||||
getSession,
|
||||
getCookieJar,
|
||||
destroyBypassSessions,
|
||||
destroyBrowserSessions,
|
||||
proxyAgent,
|
||||
};
|
||||
@@ -1,18 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const sharp = require('sharp');
|
||||
const { argv } = require('yargs');
|
||||
|
||||
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
|
||||
|
||||
async function scan() {
|
||||
console.log(url);
|
||||
|
||||
const res = await bhttp.get(url);
|
||||
const stats = await sharp(res.body).stats();
|
||||
|
||||
console.log(stats);
|
||||
}
|
||||
|
||||
scan();
|
||||
@@ -1,21 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
async function run() {
|
||||
const network = await knex('networks').where('slug', 'mofos').first();
|
||||
const sites = await knex('sites').where('network_id', network.id);
|
||||
|
||||
await Promise.map(sites, async (site) => {
|
||||
const res = await bhttp.get(site.url);
|
||||
|
||||
console.log(site.url, res.statusCode);
|
||||
}, {
|
||||
concurrency: 5,
|
||||
});
|
||||
}
|
||||
|
||||
run();
|
||||
@@ -1,37 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const fs = require('fs-extra');
|
||||
const knex = require('../knex');
|
||||
|
||||
async function init() {
|
||||
const sites = await knex('sites')
|
||||
.select('networks.name', 'sites.slug')
|
||||
.join('networks', 'networks.id', 'sites.network_id')
|
||||
.where('networks.slug', 'score');
|
||||
|
||||
await Promise.map(sites, async (site) => {
|
||||
const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
|
||||
|
||||
console.log(url);
|
||||
|
||||
const res = await bhttp.get(url, {
|
||||
responseTimeout: 5000,
|
||||
});
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
console.log(`Saving logo for ${site.slug}`);
|
||||
|
||||
await fs.writeFile(`./score/${site.slug}.png`, res.body);
|
||||
}
|
||||
|
||||
console.log(`No logo found for ${site.slug}`);
|
||||
}, {
|
||||
concurrency: 10,
|
||||
});
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,55 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const { PassThrough } = require('stream');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const bhttp = require('bhttp');
|
||||
const blake2 = require('blake2');
|
||||
const sharp = require('sharp');
|
||||
|
||||
const url = 'https://thumbs.julesjordan.com/trial/content//upload/dl03/julesjordan/oil_overload_16_scene2//photos/alina_lopez_jules_jordan_com_77.jpg';
|
||||
|
||||
async function init() {
|
||||
const hash = new blake2.Hash('blake2b');
|
||||
hash.setEncoding('hex');
|
||||
|
||||
const res = await bhttp.get(url, {
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const metaStream = sharp();
|
||||
const hashStream = new PassThrough();
|
||||
const target = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina.jpg'));
|
||||
const thumbTarget = fs.createWriteStream(path.join(config.media.path, 'temp', 'alina_thumb.jpg'));
|
||||
|
||||
hashStream.on('data', (chunk) => {
|
||||
hash.write(chunk);
|
||||
});
|
||||
|
||||
metaStream.clone()
|
||||
.resize(320)
|
||||
.pipe(thumbTarget);
|
||||
|
||||
const stream = res
|
||||
.pipe(metaStream)
|
||||
.pipe(hashStream)
|
||||
.pipe(target);
|
||||
|
||||
stream.on('finish', () => {
|
||||
hash.end();
|
||||
const digest = hash.read();
|
||||
|
||||
console.log('stream', digest);
|
||||
});
|
||||
|
||||
metaStream.on('info', (info) => {
|
||||
console.log('info', info);
|
||||
});
|
||||
|
||||
const stats = await metaStream.stats();
|
||||
|
||||
console.log('stats', stats);
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,827 +0,0 @@
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const fs = require('fs');
|
||||
|
||||
const channels = [{
|
||||
name: 'Analyzed Girls',
|
||||
slug: 'analyzedgirls',
|
||||
url: 'https://analyzedgirls.com',
|
||||
alias: ['ag',
|
||||
],
|
||||
tags: ['anal',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 178,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Archived Updates',
|
||||
slug: 'archivedupdates',
|
||||
url: 'https://archivedupdates.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 482,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Ass Teen Mouth',
|
||||
slug: 'assteenmouth',
|
||||
url: 'https://assteenmouth.com',
|
||||
alias: ['atm',
|
||||
],
|
||||
tags: ['anal',
|
||||
'atm',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 180,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Bang Teen Pussy',
|
||||
slug: 'bangteenpussy',
|
||||
url: 'https://bangteenpussy.com',
|
||||
alias: ['btp',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 182,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Brutal Invasion',
|
||||
slug: 'brutalinvasion',
|
||||
url: 'https://brutalinvasion.com',
|
||||
alias: ['bin',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 184,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Clean My Ass',
|
||||
slug: 'cleanmyass',
|
||||
url: 'https://cleanmyass.com',
|
||||
tags: ['ass-eating'],
|
||||
alias: ['cma',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 362,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'College Party Time',
|
||||
slug: 'collegepartytime',
|
||||
url: 'https://collegepartytime.com',
|
||||
alias: ['cpt',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 278,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Cumaholic Teens',
|
||||
slug: 'cumaholicteens',
|
||||
url: 'https://cumaholicteens.com',
|
||||
alias: ['ct',
|
||||
'cumoholic teens',
|
||||
'cumoholicteens',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 186,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Cum Filled Throat',
|
||||
slug: 'cumfilledthroat',
|
||||
url: 'https://cumfilledthroat.com',
|
||||
alias: ['cft',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 280,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Cum Sumption Cocktail',
|
||||
slug: 'cumsumptioncocktail',
|
||||
url: 'https://cumsumptioncocktail.com',
|
||||
alias: ['csc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 282,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Defiled18',
|
||||
slug: 'defiled18',
|
||||
url: 'https://defiled18.com',
|
||||
alias: ['d18',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 188,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Dirty Baby Sitter',
|
||||
slug: 'dirtybabysitter',
|
||||
url: 'https://dirtybabysitter.com',
|
||||
alias: ['dbs',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 284,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Double Teamed Teens',
|
||||
slug: 'doubleteamedteens',
|
||||
url: 'https://doubleteamedteens.com',
|
||||
alias: ['dtt',
|
||||
],
|
||||
tags: ['teen',
|
||||
'dp',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 190,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Dreamteens HD',
|
||||
slug: 'dreamteenshd',
|
||||
url: 'https://dreamteenshd.com',
|
||||
alias: ['dhd',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 192,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Drill Her Butt',
|
||||
slug: 'drillherbutt',
|
||||
url: 'https://drillherbutt.com',
|
||||
tags: ['anal'],
|
||||
alias: ['dhb',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 288,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Dual Throat',
|
||||
slug: 'dualthroat',
|
||||
url: 'https://dualthroat.com',
|
||||
alias: ['dt',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 290,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Gang Land Victims',
|
||||
slug: 'ganglandvictims',
|
||||
url: 'https://ganglandvictims.com',
|
||||
alias: ['glv',
|
||||
],
|
||||
tags: ['gangbang',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 292,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Girls Got Cream',
|
||||
slug: 'girlsgotcream',
|
||||
url: 'https://girlsgotcream.com',
|
||||
alias: ['ggc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 194,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Hardcore Youth',
|
||||
slug: 'hardcoreyouth',
|
||||
url: 'https://hardcoreyouth.com',
|
||||
alias: ['hcy',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 196,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Little Hellcat',
|
||||
slug: 'littlehellcat',
|
||||
url: 'https://littlehellcat.com',
|
||||
alias: ['lhc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 198,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Little Teen Suckers',
|
||||
slug: 'littleteensuckers',
|
||||
url: 'https://littleteensuckers.com',
|
||||
alias: ['lts',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 294,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'MAIKO',
|
||||
slug: 'maiko',
|
||||
url: 'https://maiko.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 566,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Make Teen Gape',
|
||||
slug: 'maketeengape',
|
||||
url: 'https://maketeengape.com',
|
||||
alias: ['mtg',
|
||||
],
|
||||
tags: ['teen',
|
||||
'gaping',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 200,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Make Teen Moan',
|
||||
slug: 'maketeenmoan',
|
||||
url: 'https://maketeenmoan.com',
|
||||
alias: ['mtm',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 296,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Mega Penetrations',
|
||||
slug: 'megapenetrations',
|
||||
url: 'https://megapenetrations.com',
|
||||
alias: ['mp',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 298,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Messy Gang Bangs',
|
||||
slug: 'messygangbangs',
|
||||
url: 'https://messygangbangs.com',
|
||||
alias: ['mgb',
|
||||
],
|
||||
tags: ['gangbang',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 300,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'My Black Coeds',
|
||||
slug: 'myblackcoeds',
|
||||
url: 'https://myblackcoeds.com',
|
||||
alias: ['mbc',
|
||||
],
|
||||
tags: ['ebony',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 302,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'My Latina Teen',
|
||||
slug: 'mylatinateen',
|
||||
url: 'https://mylatinateen.com',
|
||||
alias: ['mlt',
|
||||
],
|
||||
tags: ['teen',
|
||||
'latina',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 304,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Nasty Ass Lickers',
|
||||
slug: 'nastyasslickers',
|
||||
url: 'https://nastyasslickers.com',
|
||||
tags: ['ass-eating'],
|
||||
alias: ['nal',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 306,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Naughty Little Nymphs',
|
||||
slug: 'naughtylittlenymphs',
|
||||
url: 'https://naughtylittlenymphs.com',
|
||||
alias: ['nln',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 308,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'NC UK Sinners',
|
||||
slug: 'ncuksinners',
|
||||
url: 'https://ncuksinners.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 568,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'NC Verso Cinema',
|
||||
slug: 'ncversocinema',
|
||||
url: 'https://ncversocinema.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 570,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Network Mixed',
|
||||
slug: 'networkmixed',
|
||||
url: 'https://networkmixed.com',
|
||||
alias: ['anc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 360,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Never Done That Before',
|
||||
slug: 'neverdonethatbefore',
|
||||
url: 'https://neverdonethatbefore.com',
|
||||
alias: ['ndt',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 310,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Nylon Sweeties',
|
||||
slug: 'nylonsweeties',
|
||||
url: 'https://nylonsweeties.com',
|
||||
alias: ['nsw',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 202,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Pink Eye Sluts',
|
||||
slug: 'pinkeyesluts',
|
||||
url: 'https://pinkeyesluts.com',
|
||||
alias: ['pes',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 312,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Plug2 Holes',
|
||||
slug: 'plug2holes',
|
||||
url: 'https://plug2holes.com',
|
||||
alias: ['p2h',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 314,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Pussy Babes',
|
||||
slug: 'pussybabes',
|
||||
url: 'https://pussybabes.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 556,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Road Gang Bangs',
|
||||
slug: 'roadgangbangs',
|
||||
url: 'https://roadgangbangs.com',
|
||||
alias: ['rgb',
|
||||
],
|
||||
tags: ['gangbang',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 316,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Russian Teen Updates',
|
||||
slug: 'russianteenupdates',
|
||||
url: 'https://russianteenupdates.com',
|
||||
tags: ['teen',
|
||||
'russian',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 418,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'School Bus Chicks',
|
||||
slug: 'schoolbuschicks',
|
||||
url: 'https://schoolbuschicks.com',
|
||||
alias: ['sbc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 318,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Seductive18',
|
||||
slug: 'seductive18',
|
||||
url: 'https://seductive18.com',
|
||||
alias: ['s18',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 204,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Show Me Gape',
|
||||
slug: 'showmegape',
|
||||
url: 'https://showmegape.com',
|
||||
tags: ['gaping'],
|
||||
alias: ['smg',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 320,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Shy Teachers Pet',
|
||||
slug: 'shyteacherspet',
|
||||
url: 'https://shyteacherspet.com',
|
||||
alias: ['stp',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 322,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Small Tits Hunter',
|
||||
slug: 'smalltitshunter',
|
||||
url: 'https://smalltitshunter.com',
|
||||
alias: ['sth',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 324,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Spermantino',
|
||||
slug: 'spermantino',
|
||||
url: 'https://spermantino.com',
|
||||
alias: ['spe',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 366,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'TCC Network Archives',
|
||||
slug: 'tccnetworkarchives',
|
||||
url: 'https://tccnetworkarchives.com',
|
||||
alias: ['all',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 176,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teach My Ass',
|
||||
slug: 'teachmyass',
|
||||
url: 'https://teachmyass.com',
|
||||
tags: ['anal'],
|
||||
alias: ['tma',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 368,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teenagers Going Wild',
|
||||
slug: 'teenagersgoingwild',
|
||||
url: 'https://teenagersgoingwild.com',
|
||||
alias: ['tgw',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 326,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teen Anal Casting',
|
||||
slug: 'teenanalcasting',
|
||||
url: 'https://teenanalcasting.com',
|
||||
alias: ['tac',
|
||||
],
|
||||
tags: ['anal',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 206,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teen Drillers',
|
||||
slug: 'teendrillers',
|
||||
url: 'https://teendrillers.com',
|
||||
alias: ['td',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 208,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teen Gina',
|
||||
slug: 'teengina',
|
||||
url: 'https://teengina.com',
|
||||
alias: ['tg',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 210,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teens Love Blacks',
|
||||
slug: 'teensloveblacks',
|
||||
url: 'https://teensloveblacks.com',
|
||||
alias: ['tlb',
|
||||
],
|
||||
tags: ['interracial',
|
||||
'bbc',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 328,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teens Natural Way',
|
||||
slug: 'teensnaturalway',
|
||||
url: 'https://teensnaturalway.com',
|
||||
alias: ['tnw',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 212,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teens Try Anal',
|
||||
slug: 'teenstryanal',
|
||||
url: 'https://teenstryanal.com',
|
||||
alias: ['tta',
|
||||
],
|
||||
tags: ['anal',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 330,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teens Try Blacks',
|
||||
slug: 'teenstryblacks',
|
||||
url: 'https://teenstryblacks.com',
|
||||
alias: ['ttb',
|
||||
],
|
||||
tags: ['interracial',
|
||||
'bbc',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 214,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Teens Want Orgies',
|
||||
slug: 'teenswantorgies',
|
||||
url: 'https://teenswantorgies.com',
|
||||
alias: ['two',
|
||||
],
|
||||
tags: ['teen',
|
||||
'threesome',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 332,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Try Teens',
|
||||
slug: 'tryteens',
|
||||
url: 'https://tryteens.com',
|
||||
alias: ['tt',
|
||||
],
|
||||
tags: ['teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 216,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Tug Job Queens',
|
||||
slug: 'tugjobqueens',
|
||||
url: 'https://tugjobqueens.com',
|
||||
tags: ['handjob'],
|
||||
alias: ['tjq',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 334,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'UK Sinners',
|
||||
slug: 'uksinners',
|
||||
url: 'https://uksinners.com',
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 558,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'White Box Black Cocks',
|
||||
slug: 'whiteboxblackcocks',
|
||||
url: 'https://whiteboxblackcocks.com',
|
||||
alias: ['wbc',
|
||||
],
|
||||
tags: ['interracial',
|
||||
'bbc',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 336,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'White Teens Black Cocks',
|
||||
slug: 'whiteteensblackcocks',
|
||||
url: 'https://whiteteensblackcocks.com',
|
||||
alias: ['wbc',
|
||||
],
|
||||
tags: ['interracial',
|
||||
'bbc',
|
||||
'teen',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
siteId: 218,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Young Throats',
|
||||
slug: 'youngthroats',
|
||||
url: 'https://youngthroats.com',
|
||||
tags: ['blowjob', 'deepthroat', 'facefucking'],
|
||||
alias: ['yt',
|
||||
],
|
||||
parent: 'teencoreclub',
|
||||
parameters: {
|
||||
siteId: 220,
|
||||
},
|
||||
}];
|
||||
|
||||
async function init() {
|
||||
const results = await Promise.map(channels, async (channel) => {
|
||||
try {
|
||||
const res = await bhttp.head(channel.url, {
|
||||
followRedirects: false,
|
||||
responseTimeout: 30000,
|
||||
});
|
||||
|
||||
console.log(res.statusCode, channel.url);
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
return channel;
|
||||
}
|
||||
|
||||
throw new Error(`${channel.url} no longer available: ${res.statusCode}`);
|
||||
} catch (error) {
|
||||
console.error(channel.url, error.message);
|
||||
|
||||
return {
|
||||
name: channel.name,
|
||||
slug: channel.slug,
|
||||
url: `https://teencoreclub.com/browsevideos/site/${channel.parameters.siteId}/${channel.name.replace(/\s+/g, '')}`,
|
||||
...(channel.tags && { tags: channel.tags }),
|
||||
...(channel.alias && { alias: channel.alias }),
|
||||
parent: channel.parent,
|
||||
...(channel.hasLogo !== undefined && { hasLogo: channel.hasLogo }),
|
||||
parameters: channel.parameters,
|
||||
};
|
||||
}
|
||||
}, { concurrency: 5 });
|
||||
|
||||
await fs.promises.writeFile('./tcc.json', JSON.stringify(results, null, 4));
|
||||
|
||||
console.log('done!');
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,18 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const { VirtualConsole } = require('jsdom');
|
||||
const path = require('path');
|
||||
|
||||
const logger = require('../logger')(__filename);
|
||||
|
||||
function getVirtualConsole(filepath) {
|
||||
const virtualConsole = new VirtualConsole();
|
||||
const context = path.basename(filepath).replace(path.extname(filepath), '');
|
||||
|
||||
virtualConsole.on('error', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
|
||||
virtualConsole.on('jsdomError', (message) => logger.warn(`Error from JSDOM in ${context}: ${message}`));
|
||||
|
||||
return virtualConsole;
|
||||
}
|
||||
|
||||
module.exports = getVirtualConsole;
|
||||
@@ -1,96 +0,0 @@
|
||||
const webpack = require('webpack');
|
||||
const path = require('path');
|
||||
const { VueLoaderPlugin } = require('vue-loader');
|
||||
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
|
||||
const autoprefixer = require('autoprefixer');
|
||||
const ESLintPlugin = require('eslint-webpack-plugin');
|
||||
|
||||
module.exports = {
|
||||
entry: {
|
||||
bundle: './assets/js/main.js',
|
||||
},
|
||||
output: {
|
||||
filename: '[name].js',
|
||||
path: path.join(__dirname, 'public/js'),
|
||||
},
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
test: /\.vue$/,
|
||||
include: [
|
||||
path.resolve(__dirname, 'assets'),
|
||||
],
|
||||
loader: 'vue-loader',
|
||||
options: {
|
||||
preserveWhitespace: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
test: /\.js$/,
|
||||
exclude: /node_modules/,
|
||||
use: [
|
||||
{
|
||||
loader: 'babel-loader',
|
||||
options: {
|
||||
babelrc: false,
|
||||
plugins: [
|
||||
'@babel/plugin-proposal-object-rest-spread',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
test: /\.s?css$/,
|
||||
use: [
|
||||
MiniCssExtractPlugin.loader,
|
||||
{
|
||||
loader: 'css-loader',
|
||||
options: {
|
||||
sourceMap: true,
|
||||
url: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
loader: 'postcss-loader',
|
||||
options: {
|
||||
postcssOptions: {
|
||||
plugins: [autoprefixer],
|
||||
},
|
||||
sourceMap: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
loader: 'sass-loader',
|
||||
options: {
|
||||
sourceMap: true,
|
||||
additionalData: '@import "assets/css/_breakpoints.scss";',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
test: /\.svg/,
|
||||
use: 'raw-loader',
|
||||
},
|
||||
],
|
||||
},
|
||||
plugins: [
|
||||
new VueLoaderPlugin(),
|
||||
new ESLintPlugin(),
|
||||
new MiniCssExtractPlugin({
|
||||
filename: '../css/style.css',
|
||||
}),
|
||||
new webpack.DefinePlugin({
|
||||
VERSION: JSON.stringify(process.env.npm_package_version),
|
||||
}),
|
||||
],
|
||||
resolve: {
|
||||
alias: {
|
||||
theme: path.join(__dirname, 'assets/css/_theme.scss'),
|
||||
breakpoints: path.join(__dirname, 'assets/css/_breakpoints.scss'),
|
||||
config: path.join(__dirname, `assets/js/config/${process.env.NODE_ENV || 'default'}.js`),
|
||||
vue: 'vue/dist/vue.esm-bundler.js',
|
||||
},
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user