Updated dependencies not locked by require.
This commit is contained in:
DebaucheryLibrarian
@@ -5,11 +5,12 @@
|
|||||||
"node": true
|
"node": true
|
||||||
},
|
},
|
||||||
"parserOptions": {
|
"parserOptions": {
|
||||||
"parser": "@babel/eslint-parser",
|
|
||||||
"ecmaVersion": 2024,
|
"ecmaVersion": 2024,
|
||||||
"sourceType": "script"
|
"sourceType": "script"
|
||||||
},
|
},
|
||||||
"rules": {
|
"rules": {
|
||||||
|
"import/no-unresolved": ["error", {"ignore": ["^yargs/helpers$", "^object-merge-advanced$"]}],
|
||||||
|
"import/extensions": ["error", "ignorePackages", {"pattern": {"js": "never", "mjs": "never", "jsx": "never"}, "pathGroupOverrides": [{"pattern": "@noble/hashes/**", "action": "ignore"}]}],
|
||||||
"indent": ["error", "tab"],
|
"indent": ["error", "tab"],
|
||||||
"no-tabs": "off",
|
"no-tabs": "off",
|
||||||
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
|
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
|
||||||
|
|||||||
@@ -50,11 +50,7 @@ exclude: {
|
|||||||
},
|
},
|
||||||
```
|
```
|
||||||
|
|
||||||
### Building
|
### Thumbnails
|
||||||
To build traxxx, run the following command:
|
|
||||||
|
|
||||||
`npm run build`
|
|
||||||
|
|
||||||
To generate thumbnails for new logos and tag photos, install ImageMagick and run:
|
To generate thumbnails for new logos and tag photos, install ImageMagick and run:
|
||||||
|
|
||||||
`npm run logos-thumbs`
|
`npm run logos-thumbs`
|
||||||
|
|||||||
@@ -1,138 +0,0 @@
|
|||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
|
||||||
|
|
||||||
const scenesFields = `
|
|
||||||
id int,
|
|
||||||
title text,
|
|
||||||
title_filtered text,
|
|
||||||
shoot_id text,
|
|
||||||
channel_id int,
|
|
||||||
channel_name text,
|
|
||||||
channel_slug text,
|
|
||||||
network_id int,
|
|
||||||
network_name text,
|
|
||||||
network_slug text,
|
|
||||||
studio_id int,
|
|
||||||
studio_name text,
|
|
||||||
studio_slug text,
|
|
||||||
entity_ids multi,
|
|
||||||
actor_ids multi,
|
|
||||||
actors text,
|
|
||||||
tag_ids multi,
|
|
||||||
tags text,
|
|
||||||
movie_ids multi,
|
|
||||||
movies text,
|
|
||||||
serie_ids multi,
|
|
||||||
series text,
|
|
||||||
meta text,
|
|
||||||
date timestamp,
|
|
||||||
is_showcased bool,
|
|
||||||
created_at timestamp,
|
|
||||||
effective_date timestamp,
|
|
||||||
stashed int,
|
|
||||||
dupe_index int
|
|
||||||
`;
|
|
||||||
|
|
||||||
const moviesFields = `
|
|
||||||
id int,
|
|
||||||
title text,
|
|
||||||
title_filtered text,
|
|
||||||
channel_id int,
|
|
||||||
channel_name text,
|
|
||||||
channel_slug text,
|
|
||||||
network_id int,
|
|
||||||
network_name text,
|
|
||||||
network_slug text,
|
|
||||||
entity_ids multi,
|
|
||||||
actor_ids multi,
|
|
||||||
actors text,
|
|
||||||
tag_ids multi,
|
|
||||||
tags text,
|
|
||||||
meta text,
|
|
||||||
date timestamp,
|
|
||||||
has_cover bool,
|
|
||||||
created_at timestamp,
|
|
||||||
effective_date timestamp,
|
|
||||||
stashed int,
|
|
||||||
stashed_scenes int,
|
|
||||||
stashed_total int,
|
|
||||||
dupe_index int
|
|
||||||
`;
|
|
||||||
|
|
||||||
const actorsFields = `
|
|
||||||
id int,
|
|
||||||
name text,
|
|
||||||
slug string,
|
|
||||||
gender string,
|
|
||||||
date_of_birth timestamp,
|
|
||||||
country string,
|
|
||||||
has_avatar bool,
|
|
||||||
mass int,
|
|
||||||
height int,
|
|
||||||
cup string,
|
|
||||||
natural_boobs int,
|
|
||||||
penis_length int,
|
|
||||||
penis_girth int,
|
|
||||||
stashed int,
|
|
||||||
scenes int
|
|
||||||
`;
|
|
||||||
|
|
||||||
exports.up = async () => {
|
|
||||||
try {
|
|
||||||
await utilsApi.sql(`create table if not exists scenes (${scenesFields})`);
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
|
||||||
scene_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists movies (${moviesFields})`);
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
|
||||||
movie_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`);
|
|
||||||
|
|
||||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
|
||||||
actor_id int,
|
|
||||||
stash_id int,
|
|
||||||
user_id int,
|
|
||||||
created_at timestamp
|
|
||||||
)`);
|
|
||||||
|
|
||||||
/*
|
|
||||||
await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id'));
|
|
||||||
await knex.schema.alterTable('stashes_movies', (table) => table.increments('id'));
|
|
||||||
await knex.schema.alterTable('stashes_actors', (table) => table.increments('id'));
|
|
||||||
await knex.schema.alterTable('stashes_series', (table) => table.increments('id'));
|
|
||||||
*/
|
|
||||||
} catch (error) {
|
|
||||||
console.log(error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
exports.down = async (knex) => {
|
|
||||||
await utilsApi.sql('drop table if exists scenes');
|
|
||||||
await utilsApi.sql('drop table if exists scenes_stashed');
|
|
||||||
await utilsApi.sql('drop table if exists movies');
|
|
||||||
await utilsApi.sql('drop table if exists movies_stashed');
|
|
||||||
await utilsApi.sql('drop table if exists actors');
|
|
||||||
await utilsApi.sql('drop table if exists actors_stashed');
|
|
||||||
|
|
||||||
await knex.schema.alterTable('stashes_scenes', (table) => table.dropColumn('id'));
|
|
||||||
await knex.schema.alterTable('stashes_movies', (table) => table.dropColumn('id'));
|
|
||||||
await knex.schema.alterTable('stashes_actors', (table) => table.dropColumn('id'));
|
|
||||||
await knex.schema.alterTable('stashes_series', (table) => table.dropColumn('id'));
|
|
||||||
};
|
|
||||||
@@ -216,12 +216,7 @@ module.exports = {
|
|||||||
},
|
},
|
||||||
bypass: {
|
bypass: {
|
||||||
browser: {
|
browser: {
|
||||||
enable: false,
|
|
||||||
clientRetirement: 20,
|
clientRetirement: 20,
|
||||||
hostnames: [ // these can run in the same browser session
|
|
||||||
'www.kink.com',
|
|
||||||
'store2.psmcdn.net', // Team Skeet API
|
|
||||||
],
|
|
||||||
remote: {
|
remote: {
|
||||||
enable: false,
|
enable: false,
|
||||||
use: false,
|
use: false,
|
||||||
@@ -229,15 +224,6 @@ module.exports = {
|
|||||||
key: null,
|
key: null,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
cloudflare: {
|
|
||||||
enable: false,
|
|
||||||
auto: true, // try bypass when CF challenge is detected
|
|
||||||
path: 'http://localhost:8191/v1',
|
|
||||||
sharedHostnames: [ // these can run in the same browser session
|
|
||||||
'store2.psmcdn.net', // Team Skeet API
|
|
||||||
],
|
|
||||||
independentHostnames: [], // these must run in their own browser session
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
limits: {
|
limits: {
|
||||||
default: {
|
default: {
|
||||||
|
|||||||
Generated
+4682
-16581
File diff suppressed because it is too large
Load Diff
+26
-105
@@ -8,12 +8,6 @@
|
|||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node -r source-map-support/register src/init.js",
|
"start": "node -r source-map-support/register src/init.js",
|
||||||
"webpack": "webpack --env=production --mode=production",
|
|
||||||
"webpack-dev": "webpack --env=development --mode=development",
|
|
||||||
"webpack-watch": "webpack --progress --color --watch --env=development --mode=development",
|
|
||||||
"babel": "babel src --source-maps -d dist",
|
|
||||||
"babel-watch": "babel src -w --source-maps -d dist",
|
|
||||||
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
|
|
||||||
"eslint": "eslint src/",
|
"eslint": "eslint src/",
|
||||||
"eslint-watch": "esw --watch src/",
|
"eslint-watch": "esw --watch src/",
|
||||||
"knex": "knex",
|
"knex": "knex",
|
||||||
@@ -41,134 +35,61 @@
|
|||||||
"author": "DebaucheryLibrarian",
|
"author": "DebaucheryLibrarian",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/cli": "^7.23.4",
|
|
||||||
"@babel/core": "^7.23.3",
|
|
||||||
"@babel/eslint-parser": "^7.23.3",
|
|
||||||
"@babel/plugin-proposal-optional-chaining": "^7.8.3",
|
|
||||||
"@babel/preset-env": "^7.23.3",
|
|
||||||
"@babel/register": "^7.22.15",
|
|
||||||
"autoprefixer": "^10.4.16",
|
|
||||||
"babel-loader": "^9.1.3",
|
|
||||||
"babel-preset-airbnb": "^5.0.0",
|
|
||||||
"css-loader": "^6.8.1",
|
|
||||||
"eslint": "^8.54.0",
|
"eslint": "^8.54.0",
|
||||||
"eslint-config-airbnb": "^19.0.4",
|
|
||||||
"eslint-config-airbnb-base": "^15.0.0",
|
"eslint-config-airbnb-base": "^15.0.0",
|
||||||
"eslint-plugin-import": "^2.29.0",
|
"eslint-plugin-import": "^2.29.0",
|
||||||
"eslint-plugin-jsx-a11y": "^6.8.0",
|
"eslint-watch": "^8.0.0"
|
||||||
"eslint-plugin-react": "^7.33.2",
|
|
||||||
"eslint-plugin-vue": "^9.18.1",
|
|
||||||
"eslint-watch": "^8.0.0",
|
|
||||||
"eslint-webpack-plugin": "^4.0.1",
|
|
||||||
"mini-css-extract-plugin": "^2.7.6",
|
|
||||||
"postcss-loader": "^7.3.3",
|
|
||||||
"raw-loader": "^4.0.2",
|
|
||||||
"sass-loader": "^13.3.2",
|
|
||||||
"style-loader": "^3.3.3",
|
|
||||||
"vue-loader": "^17.3.1",
|
|
||||||
"webpack": "^5.89.0",
|
|
||||||
"webpack-cli": "^5.1.4"
|
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.458.0",
|
"@aws-sdk/client-s3": "^3.458.0",
|
||||||
"@aws-sdk/credential-provider-ini": "^3.458.0",
|
|
||||||
"@aws-sdk/lib-storage": "^3.458.0",
|
"@aws-sdk/lib-storage": "^3.458.0",
|
||||||
"@casl/ability": "^6.5.0",
|
"@faker-js/faker": "^10.6.0",
|
||||||
"@faker-js/faker": "^8.3.1",
|
"@noble/hashes": "^2.4.0",
|
||||||
"@graphile-contrib/pg-order-by-related": "^1.0.0",
|
|
||||||
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
|
||||||
"@graphile/pg-aggregates": "^0.1.1",
|
|
||||||
"@noble/hashes": "^1.8.0",
|
|
||||||
"@rsc-parser/react-client": "^1.1.2",
|
"@rsc-parser/react-client": "^1.1.2",
|
||||||
|
"@tensorflow/tfjs-core": "4.22.0",
|
||||||
"@tensorflow/tfjs-node": "^4.22.0",
|
"@tensorflow/tfjs-node": "^4.22.0",
|
||||||
"@vladmandic/human": "^3.3.6",
|
"@vladmandic/human": "^3.3.6",
|
||||||
"acorn": "^8.11.2",
|
|
||||||
"array-equal": "^1.0.2",
|
|
||||||
"babel-polyfill": "^6.26.0",
|
|
||||||
"bhttp": "^1.2.8",
|
"bhttp": "^1.2.8",
|
||||||
"bluebird": "^3.7.2",
|
"bluebird": "^3.7.2",
|
||||||
"body-parser": "^1.20.2",
|
|
||||||
"bottleneck": "^2.19.5",
|
"bottleneck": "^2.19.5",
|
||||||
"casual": "^1.6.2",
|
"casual": "^1.6.2",
|
||||||
"cheerio": "^1.0.0-rc.12",
|
|
||||||
"cli-confirm": "^1.0.1",
|
"cli-confirm": "^1.0.1",
|
||||||
"clipboardy": "^2.3.0",
|
"config": "^5.0.1",
|
||||||
"cloudscraper": "^4.6.0",
|
"convert": "^7.0.2",
|
||||||
"config": "^3.3.12",
|
|
||||||
"connect-session-knex": "^4.0.0",
|
|
||||||
"convert": "^4.14.0",
|
|
||||||
"cookie": "^0.6.0",
|
|
||||||
"css": "^3.0.0",
|
|
||||||
"csv-parse": "^6.1.0",
|
|
||||||
"csv-stringify": "^6.4.4",
|
"csv-stringify": "^6.4.4",
|
||||||
"date-fns": "^2.30.0",
|
"date-fns": "^4.4.0",
|
||||||
"dayjs": "^1.11.10",
|
|
||||||
"dompurify": "^3.0.6",
|
"dompurify": "^3.0.6",
|
||||||
"ejs": "^3.1.9",
|
"escape-string-regexp": "^5.0.0",
|
||||||
"escape-string-regexp": "^4.0.0",
|
|
||||||
"express": "^4.18.2",
|
|
||||||
"express-promise-router": "^4.1.1",
|
|
||||||
"express-query-boolean": "^2.0.0",
|
|
||||||
"express-react-views": "^0.11.0",
|
|
||||||
"express-session": "^1.17.3",
|
|
||||||
"face-api.js": "^0.22.2",
|
|
||||||
"file-type": "^18.7.0",
|
|
||||||
"fluent-ffmpeg": "^2.1.3",
|
"fluent-ffmpeg": "^2.1.3",
|
||||||
"fs-extra": "^11.1.1",
|
"fs-extra": "^11.1.1",
|
||||||
"graphile-build": "^4.14.0",
|
|
||||||
"graphile-utils": "^4.14.0",
|
|
||||||
"graphql": "^15.8.0",
|
|
||||||
"html-entities": "^2.4.0",
|
"html-entities": "^2.4.0",
|
||||||
"https-proxy-agent": "^7.0.5",
|
"inquirer": "^14.2.5",
|
||||||
"iconv-lite": "^0.6.3",
|
|
||||||
"inquirer": "^8.2.6",
|
|
||||||
"inspector-api": "^1.4.8",
|
"inspector-api": "^1.4.8",
|
||||||
"jsdom": "^23.0.1",
|
"jsdom": "^31.0.0",
|
||||||
"knex": "^3.0.1",
|
"knex": "^3.0.1",
|
||||||
"knex-migrate": "^1.7.4",
|
"knex-migrate": "^1.7.4",
|
||||||
"longjohn": "^0.2.12",
|
"mime": "^4.1.2",
|
||||||
"manticoresearch": "^4.0.0",
|
|
||||||
"merge-anything": "^5.1.7",
|
|
||||||
"mime": "^3.0.0",
|
|
||||||
"mitt": "^3.0.1",
|
|
||||||
"moment": "^2.29.4",
|
"moment": "^2.29.4",
|
||||||
"mysql": "^2.18.1",
|
"nanoid": "^6.0.2",
|
||||||
"nanoid": "^3.3.7",
|
|
||||||
"node-fetch": "^3.3.2",
|
"node-fetch": "^3.3.2",
|
||||||
"npm": "^11.7.0",
|
"object-merge-advanced": "^14.2.5",
|
||||||
"object-merge-advanced": "^12.1.0",
|
|
||||||
"object.omit": "^3.0.0",
|
"object.omit": "^3.0.0",
|
||||||
"pg": "^8.11.3",
|
"pg": "^8.11.3",
|
||||||
"playwright": "^1.57.0",
|
|
||||||
"playwright-core": "^1.57.0",
|
|
||||||
"postgraphile": "^4.14.0",
|
|
||||||
"postgraphile-plugin-connection-filter": "^2.3.0",
|
|
||||||
"promise-task-queue": "^1.2.0",
|
"promise-task-queue": "^1.2.0",
|
||||||
"prop-types": "^15.8.1",
|
"redis": "^6.3.1",
|
||||||
"puppeteer": "^24.4.0",
|
"sharp": "^0.35.5",
|
||||||
"puppeteer-extra": "^3.3.6",
|
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
||||||
"redis": "^4.6.11",
|
|
||||||
"sharp": "^0.33.0-rc.2",
|
|
||||||
"showdown": "^2.1.0",
|
|
||||||
"source-map-support": "^0.5.21",
|
"source-map-support": "^0.5.21",
|
||||||
"template-format": "^1.2.5",
|
"template-format": "^2.0.0",
|
||||||
"tippy.js": "^6.3.7",
|
"undici": "^8.12.4",
|
||||||
"tough-cookie": "^4.1.3",
|
"unprint": "^1.0.0-beta.0",
|
||||||
"tunnel": "0.0.6",
|
"why-is-node-running": "^3.2.3",
|
||||||
"ua-parser-js": "^1.0.37",
|
|
||||||
"undici": "^7.24.7",
|
|
||||||
"unprint": "^0.19.20",
|
|
||||||
"url-pattern": "^1.0.3",
|
|
||||||
"v-tooltip": "^2.1.3",
|
|
||||||
"video.js": "^8.6.1",
|
|
||||||
"videojs-vr": "^2.0.0",
|
|
||||||
"vue": "^3.3.9",
|
|
||||||
"vue-router": "^4.2.5",
|
|
||||||
"vuex": "^4.1.0",
|
|
||||||
"why-is-node-running": "^2.2.2",
|
|
||||||
"winston": "^3.11.0",
|
"winston": "^3.11.0",
|
||||||
"winston-daily-rotate-file": "^4.7.1",
|
"winston-daily-rotate-file": "^5.0.0",
|
||||||
"yargs": "^17.7.2"
|
"yargs": "^18.2.0"
|
||||||
|
},
|
||||||
|
"allowScripts": {
|
||||||
|
"sharp": true,
|
||||||
|
"@tensorflow/tfjs-node": true,
|
||||||
|
"core-js": false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const casual = require('casual');
|
const casual = require('casual');
|
||||||
const fs = require('fs-extra');
|
const fs = require('fs-extra');
|
||||||
const nanoid = require('nanoid');
|
const { nanoid } = require('nanoid');
|
||||||
const stringify = Promise.promisify(require('csv-stringify'));
|
const stringify = Promise.promisify(require('csv-stringify'));
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -4,11 +4,11 @@ const config = require('config');
|
|||||||
const util = require('util');
|
const util = require('util');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
const { blake2b } = require('@noble/hashes/blake2b');
|
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||||
const DOMPurify = require('dompurify');
|
const DOMPurify = require('dompurify');
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const omit = require('object.omit');
|
const omit = require('object.omit');
|
||||||
const inquirer = require('inquirer');
|
const inquirer = require('inquirer').default;
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
|
|
||||||
const { window } = new JSDOM('');
|
const { window } = new JSDOM('');
|
||||||
@@ -413,7 +413,7 @@ async function curateProfile(profile, actor) {
|
|||||||
|
|
||||||
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
|
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
|
||||||
|
|
||||||
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(slugify(curatedProfile.description), { dkLen: 24 })).toString('hex');
|
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(Buffer.from(slugify(curatedProfile.description)), { dkLen: 24 })).toString('hex');
|
||||||
|
|
||||||
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
||||||
|
|
||||||
|
|||||||
+2
-4
@@ -5,7 +5,7 @@ const startTime = new Date();
|
|||||||
const config = require('config');
|
const config = require('config');
|
||||||
const util = require('util');
|
const util = require('util');
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
const log = require('why-is-node-running');
|
const log = require('why-is-node-running').default;
|
||||||
const Inspector = require('inspector-api');
|
const Inspector = require('inspector-api');
|
||||||
const fs = require('fs').promises;
|
const fs = require('fs').promises;
|
||||||
const { format, intervalToDuration } = require('date-fns');
|
const { format, intervalToDuration } = require('date-fns');
|
||||||
@@ -269,12 +269,10 @@ async function init() {
|
|||||||
logger.error(error);
|
logger.error(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
await http.destroyBypassSessions();
|
|
||||||
await http.destroyBrowserSessions();
|
|
||||||
await unprint.closeAllBrowsers();
|
await unprint.closeAllBrowsers();
|
||||||
|
|
||||||
knex.destroy();
|
knex.destroy();
|
||||||
redis.disconnect();
|
redis.destroy();
|
||||||
|
|
||||||
done = true;
|
done = true;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-1
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const yargs = require('yargs');
|
const yargs = require('yargs');
|
||||||
|
const { hideBin } = require('yargs/helpers');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
function interpretDate(after, ignoreIfEmpty = false) {
|
function interpretDate(after, ignoreIfEmpty = false) {
|
||||||
@@ -27,7 +28,7 @@ function interpretDate(after, ignoreIfEmpty = false) {
|
|||||||
.toDate();
|
.toDate();
|
||||||
}
|
}
|
||||||
|
|
||||||
const { argv } = yargs
|
const { argv } = yargs(hideBin(process.argv))
|
||||||
.command('npm start')
|
.command('npm start')
|
||||||
.command('update', 'fetch latest updates', () => {
|
.command('update', 'fetch latest updates', () => {
|
||||||
console.log('UPDATING!');
|
console.log('UPDATING!');
|
||||||
|
|||||||
-110
@@ -1,110 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
const util = require('util');
|
|
||||||
const crypto = require('crypto');
|
|
||||||
|
|
||||||
const knex = require('./knex');
|
|
||||||
const { curateUser, fetchUser } = require('./users');
|
|
||||||
const { HttpError } = require('./errors');
|
|
||||||
|
|
||||||
const scrypt = util.promisify(crypto.scrypt);
|
|
||||||
|
|
||||||
async function verifyPassword(password, storedPassword) {
|
|
||||||
const [salt, hash] = storedPassword.split('/');
|
|
||||||
const hashedPassword = (await scrypt(password, salt, 64)).toString('hex');
|
|
||||||
|
|
||||||
if (hashedPassword === hash) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new HttpError('Username or password incorrect', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function login(credentials) {
|
|
||||||
if (!config.auth.login) {
|
|
||||||
throw new HttpError('Authentication is disabled', 405);
|
|
||||||
}
|
|
||||||
|
|
||||||
const user = await fetchUser(credentials.username.trim(), true);
|
|
||||||
|
|
||||||
if (!user) {
|
|
||||||
throw new HttpError('Username or password incorrect', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
await verifyPassword(credentials.password, user.password);
|
|
||||||
|
|
||||||
await knex('users')
|
|
||||||
.update('last_login', 'NOW()')
|
|
||||||
.where('id', user.id);
|
|
||||||
|
|
||||||
return curateUser(user);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function signup(credentials) {
|
|
||||||
if (!config.auth.signup) {
|
|
||||||
throw new HttpError('Authentication is disabled', 405);
|
|
||||||
}
|
|
||||||
|
|
||||||
const curatedUsername = credentials.username.trim();
|
|
||||||
|
|
||||||
if (!curatedUsername) {
|
|
||||||
throw new HttpError('Username required', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (curatedUsername.length < config.auth.usernameLength[0]) {
|
|
||||||
throw new HttpError('Username is too short', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (curatedUsername.length > config.auth.usernameLength[1]) {
|
|
||||||
throw new HttpError('Username is too long', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!config.auth.usernamePattern.test(curatedUsername)) {
|
|
||||||
throw new HttpError('Username contains invalid characters', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!credentials.email) {
|
|
||||||
throw new HttpError('E-mail required', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (credentials.password?.length < 3) {
|
|
||||||
throw new HttpError('Password must be 3 characters or longer', 400);
|
|
||||||
}
|
|
||||||
|
|
||||||
const existingUser = await knex('users')
|
|
||||||
.where('username', curatedUsername)
|
|
||||||
.orWhere('email', credentials.email)
|
|
||||||
.first();
|
|
||||||
|
|
||||||
if (existingUser) {
|
|
||||||
throw new HttpError('Username or e-mail already in use', 409);
|
|
||||||
}
|
|
||||||
|
|
||||||
const salt = crypto.randomBytes(16).toString('hex');
|
|
||||||
const hashedPassword = (await scrypt(credentials.password, salt, 64)).toString('hex');
|
|
||||||
const storedPassword = `${salt}/${hashedPassword}`;
|
|
||||||
|
|
||||||
const [{ id: userId }] = await knex('users')
|
|
||||||
.insert({
|
|
||||||
username: curatedUsername,
|
|
||||||
email: credentials.email,
|
|
||||||
password: storedPassword,
|
|
||||||
})
|
|
||||||
.returning('id');
|
|
||||||
|
|
||||||
await knex('stashes').insert({
|
|
||||||
user_id: userId,
|
|
||||||
name: 'Favorites',
|
|
||||||
slug: 'favorites',
|
|
||||||
public: false,
|
|
||||||
primary: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
return fetchUser(userId);
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
login,
|
|
||||||
signup,
|
|
||||||
};
|
|
||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const inquirer = require('inquirer');
|
const inquirer = require('inquirer').default;
|
||||||
|
|
||||||
const logger = require('./logger')(__filename);
|
const logger = require('./logger')(__filename);
|
||||||
const argv = require('./argv');
|
const argv = require('./argv');
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
require('babel-polyfill');
|
|
||||||
const init = require('./app');
|
const init = require('./app');
|
||||||
|
|
||||||
init();
|
init();
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
const config = require('config');
|
|
||||||
const manticore = require('manticoresearch');
|
|
||||||
|
|
||||||
const mantiClient = new manticore.ApiClient();
|
|
||||||
|
|
||||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
searchApi: new manticore.SearchApi(mantiClient),
|
|
||||||
indexApi: new manticore.IndexApi(mantiClient),
|
|
||||||
utilsApi: new manticore.UtilsApi(),
|
|
||||||
};
|
|
||||||
+2
-2
@@ -8,11 +8,11 @@ const fsPromises = require('fs').promises;
|
|||||||
const path = require('path');
|
const path = require('path');
|
||||||
const stream = require('stream');
|
const stream = require('stream');
|
||||||
const { nanoid } = require('nanoid/non-secure');
|
const { nanoid } = require('nanoid/non-secure');
|
||||||
const mime = require('mime');
|
const mime = require('mime').default;
|
||||||
// const fileType = require('file-type');
|
// const fileType = require('file-type');
|
||||||
const ffmpeg = require('fluent-ffmpeg');
|
const ffmpeg = require('fluent-ffmpeg');
|
||||||
const sharp = require('sharp');
|
const sharp = require('sharp');
|
||||||
const { blake2b } = require('@noble/hashes/blake2b');
|
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||||
const { format } = require('date-fns');
|
const { format } = require('date-fns');
|
||||||
const taskQueue = require('promise-task-queue');
|
const taskQueue = require('promise-task-queue');
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const inquirer = require('inquirer');
|
const inquirer = require('inquirer').default;
|
||||||
|
|
||||||
const logger = require('./logger')(__filename);
|
const logger = require('./logger')(__filename);
|
||||||
const knex = require('./knex');
|
const knex = require('./knex');
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { blake2b } = require('@noble/hashes/blake2b');
|
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||||
|
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
|
|
||||||
function getHash(identifier) {
|
function getHash(identifier) {
|
||||||
return Buffer.from(blake2b(identifier, { dkLen: 8 })).toString('hex');
|
return Buffer.from(blake2b(Buffer.from(identifier), { dkLen: 8 })).toString('hex');
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractMaleModelsFromTags(tagContainer) {
|
function extractMaleModelsFromTags(tagContainer) {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
const mime = require('mime');
|
const mime = require('mime').default;
|
||||||
|
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const { convert } = require('../utils/convert');
|
const { convert } = require('../utils/convert');
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
const escapeRegexp = require('escape-string-regexp');
|
const escapeRegexp = require('escape-string-regexp').default;
|
||||||
|
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const capitalize = require('../utils/capitalize');
|
const capitalize = require('../utils/capitalize');
|
||||||
|
|||||||
+14
-29
@@ -47,11 +47,7 @@ function scrapeAll(scenes, tilesByEntryId, channel) {
|
|||||||
|
|
||||||
release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
|
release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
release.poster = query.background('.player');
|
||||||
|
|
||||||
if (posterBackground?.background) {
|
|
||||||
release.poster = posterBackground.background.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
|
release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
|
||||||
unprint.query.url(linkEl, null),
|
unprint.query.url(linkEl, null),
|
||||||
@@ -121,8 +117,7 @@ function scrapeAllTiles(tiles, channel) {
|
|||||||
|
|
||||||
release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
|
release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
|
||||||
|
|
||||||
const posterBackground = query.style('.scene-img');
|
const posterUrl = query.background('.scene-img');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
@@ -199,8 +194,7 @@ function scrapeAllCospuri(scenes, channel) {
|
|||||||
|
|
||||||
release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
|
release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
|
||||||
|
|
||||||
const posterBackground = query.style('.scene-thumb');
|
const posterUrl = query.background('.scene-thumb');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
@@ -260,8 +254,7 @@ function scrapeAllFellatio(scenes, channel) {
|
|||||||
|
|
||||||
release.tags = query.contents('.tour-data a[href*="tag/"]');
|
release.tags = query.contents('.tour-data a[href*="tag/"]');
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -302,8 +295,7 @@ function scrapeAllHandjob(scenes, _channel) {
|
|||||||
|
|
||||||
release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
|
release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -348,8 +340,7 @@ function scrapeAllLegs(scenes, channel) {
|
|||||||
|
|
||||||
release.tags = query.contents('a[href*="tag/"]');
|
release.tags = query.contents('a[href*="tag/"]');
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -385,8 +376,7 @@ function scrapeAllFacefuck(scenes) {
|
|||||||
release.description = query.content('.infotxt');
|
release.description = query.content('.infotxt');
|
||||||
release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
|
release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -425,8 +415,7 @@ function scrapeAllTrans(scenes) {
|
|||||||
release.duration = query.duration('.sample-info');
|
release.duration = query.duration('.sample-info');
|
||||||
release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
|
release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
@@ -436,7 +425,7 @@ function scrapeAllTrans(scenes) {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
release.photos = curatePhotos(albumContext?.query.styles('.sample-lg, .sample-thumb').map((style) => style['background-image']?.match(/url\((.*)\)/)?.[1]));
|
release.photos = curatePhotos(albumContext?.query.backgrounds('.sample-lg, .sample-thumb'));
|
||||||
release.trailer = query.video();
|
release.trailer = query.video();
|
||||||
|
|
||||||
release.entryId = entryIdFromMedia(release);
|
release.entryId = entryIdFromMedia(release);
|
||||||
@@ -486,8 +475,7 @@ function scrapeAllLesbianTiles(scenes, channel) {
|
|||||||
|
|
||||||
release.tags = query.contents('.content-tags a');
|
release.tags = query.contents('.content-tags a');
|
||||||
|
|
||||||
const posterBackground = query.style('.vidthumb');
|
const posterUrl = query.background('.vidthumb');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
@@ -522,8 +510,7 @@ function scrapeAllLesbian(scenes, channel, tiles) {
|
|||||||
release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
|
release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
|
||||||
release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
|
release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -650,8 +637,7 @@ function scrapeSceneBuffet({ query }, { url, entity }) {
|
|||||||
|
|
||||||
release.tags = query.contents('.tag-list a');
|
release.tags = query.contents('.tag-list a');
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
@@ -696,8 +682,7 @@ function scrapeSceneCospuri({ query }, { url, entity }) {
|
|||||||
|
|
||||||
release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
|
release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
|
||||||
|
|
||||||
const posterBackground = query.style('.player');
|
const posterUrl = query.background('.player');
|
||||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
|
||||||
|
|
||||||
if (posterUrl) {
|
if (posterUrl) {
|
||||||
release.poster = posterUrl;
|
release.poster = posterUrl;
|
||||||
@@ -788,7 +773,7 @@ function scrapeProfile({ query }, _channel, url) {
|
|||||||
|
|
||||||
const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
|
const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
|
||||||
|| query.img('.costume-bg', { attribute: 'data-img' })
|
|| query.img('.costume-bg', { attribute: 'data-img' })
|
||||||
|| query.style('.model-profile, #profile, .carousel-item')?.['background-image']?.match(/url\((.*)\)/)?.[1];
|
|| query.background('.model-profile, #profile, .carousel-item');
|
||||||
|
|
||||||
if (avatar) {
|
if (avatar) {
|
||||||
profile.avatar = Array.from(new Set([
|
profile.avatar = Array.from(new Set([
|
||||||
|
|||||||
@@ -241,8 +241,8 @@ function generateActors(release) {
|
|||||||
: Math.floor(Math.random() * 3) + 2;
|
: Math.floor(Math.random() * 3) + 2;
|
||||||
|
|
||||||
return Array.from({ length }, () => ({
|
return Array.from({ length }, () => ({
|
||||||
name: faker.name
|
name: faker.person
|
||||||
.findName()
|
.fullName()
|
||||||
.split(' ')
|
.split(' ')
|
||||||
.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
|
.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
|
||||||
.join(' '),
|
.join(' '),
|
||||||
|
|||||||
-312
@@ -1,312 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const config = require('config');
|
|
||||||
|
|
||||||
const knex = require('./knex');
|
|
||||||
const { indexApi } = require('./manticore');
|
|
||||||
const { HttpError } = require('./errors');
|
|
||||||
const slugify = require('./utils/slugify');
|
|
||||||
const logger = require('./logger')(__filename);
|
|
||||||
|
|
||||||
let lastActorsViewRefresh = 0;
|
|
||||||
|
|
||||||
function curateStash(stash) {
|
|
||||||
if (!stash) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const curatedStash = {
|
|
||||||
id: stash.id,
|
|
||||||
name: stash.name,
|
|
||||||
slug: stash.slug,
|
|
||||||
primary: stash.primary,
|
|
||||||
};
|
|
||||||
|
|
||||||
return curatedStash;
|
|
||||||
}
|
|
||||||
|
|
||||||
function curateStashEntry(stash, user) {
|
|
||||||
const curatedStashEntry = {
|
|
||||||
user_id: user.id,
|
|
||||||
name: stash.name,
|
|
||||||
slug: slugify(stash.name),
|
|
||||||
public: false,
|
|
||||||
};
|
|
||||||
|
|
||||||
return curatedStashEntry;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchStash(stashId, sessionUser) {
|
|
||||||
if (!sessionUser) {
|
|
||||||
throw new HttpError('You are not authenthicated', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
const stash = await knex('stashes')
|
|
||||||
.where({
|
|
||||||
id: stashId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
})
|
|
||||||
.first();
|
|
||||||
|
|
||||||
if (!stash) {
|
|
||||||
throw new HttpError('You are not authorized to access this stash', 403);
|
|
||||||
}
|
|
||||||
|
|
||||||
return curateStash(stash);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchStashes(domain, itemId, sessionUser) {
|
|
||||||
const stashes = await knex(`stashes_${domain}s`)
|
|
||||||
.select('stashes.*')
|
|
||||||
.where({
|
|
||||||
[`${domain}_id`]: itemId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
})
|
|
||||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
|
||||||
|
|
||||||
return stashes.map((stash) => curateStash(stash));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function createStash(newStash, sessionUser) {
|
|
||||||
if (!sessionUser) {
|
|
||||||
throw new HttpError('You are not authenthicated', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const stash = await knex('stashes')
|
|
||||||
.insert(curateStashEntry(newStash, sessionUser))
|
|
||||||
.returning('*');
|
|
||||||
|
|
||||||
return curateStash(stash);
|
|
||||||
} catch (error) {
|
|
||||||
if (error.routine === '_bt_check_unique') {
|
|
||||||
throw new HttpError('Stash name should be unique', 409);
|
|
||||||
}
|
|
||||||
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateStash(stashId, newStash, sessionUser) {
|
|
||||||
if (!sessionUser) {
|
|
||||||
throw new HttpError('You are not authenthicated', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
const stash = await knex('stashes')
|
|
||||||
.where({
|
|
||||||
id: stashId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
})
|
|
||||||
.update(newStash)
|
|
||||||
.returning('*');
|
|
||||||
|
|
||||||
if (!stash) {
|
|
||||||
throw new HttpError('You are not authorized to modify this stash', 403);
|
|
||||||
}
|
|
||||||
|
|
||||||
return curateStash(stash);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function removeStash(stashId, sessionUser) {
|
|
||||||
if (!sessionUser) {
|
|
||||||
throw new HttpError('You are not authenthicated', 401);
|
|
||||||
}
|
|
||||||
|
|
||||||
const removed = await knex('stashes')
|
|
||||||
.where({
|
|
||||||
id: stashId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
primary: false,
|
|
||||||
})
|
|
||||||
.delete();
|
|
||||||
|
|
||||||
if (removed === 0) {
|
|
||||||
throw new HttpError('Unable to remove this stash', 400);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function refreshActorsView() {
|
|
||||||
if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
|
|
||||||
// don't refresh actors view more than once an hour
|
|
||||||
lastActorsViewRefresh = new Date();
|
|
||||||
|
|
||||||
logger.debug('Refreshing actors view');
|
|
||||||
|
|
||||||
return knex.schema.refreshMaterializedView('actors_meta');
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.silly('Skipping actors view refresh');
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function stashActor(actorId, stashId, sessionUser) {
|
|
||||||
const stash = await fetchStash(stashId, sessionUser);
|
|
||||||
|
|
||||||
const [stashed] = await knex('stashes_actors')
|
|
||||||
.insert({
|
|
||||||
stash_id: stash.id,
|
|
||||||
actor_id: actorId,
|
|
||||||
})
|
|
||||||
.returning(['id', 'created_at']);
|
|
||||||
|
|
||||||
await indexApi.replace({
|
|
||||||
index: 'actors_stashed',
|
|
||||||
id: stashed.id,
|
|
||||||
doc: {
|
|
||||||
actor_id: actorId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
stash_id: stashId,
|
|
||||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
refreshActorsView();
|
|
||||||
|
|
||||||
return fetchStashes('actor', actorId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function stashScene(sceneId, stashId, sessionUser) {
|
|
||||||
const stash = await fetchStash(stashId, sessionUser);
|
|
||||||
|
|
||||||
const [stashed] = await knex('stashes_scenes')
|
|
||||||
.insert({
|
|
||||||
stash_id: stash.id,
|
|
||||||
scene_id: sceneId,
|
|
||||||
})
|
|
||||||
.returning(['id', 'created_at']);
|
|
||||||
|
|
||||||
await indexApi.replace({
|
|
||||||
index: 'scenes_stashed',
|
|
||||||
id: stashed.id,
|
|
||||||
doc: {
|
|
||||||
// ...doc.replace.doc,
|
|
||||||
scene_id: sceneId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
stash_id: stashId,
|
|
||||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return fetchStashes('scene', sceneId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function stashMovie(movieId, stashId, sessionUser) {
|
|
||||||
const stash = await fetchStash(stashId, sessionUser);
|
|
||||||
|
|
||||||
const [stashed] = await knex('stashes_movies')
|
|
||||||
.insert({
|
|
||||||
stash_id: stash.id,
|
|
||||||
movie_id: movieId,
|
|
||||||
})
|
|
||||||
.returning(['id', 'created_at']);
|
|
||||||
|
|
||||||
await indexApi.replace({
|
|
||||||
index: 'movies_stashed',
|
|
||||||
id: stashed.id,
|
|
||||||
doc: {
|
|
||||||
movie_id: movieId,
|
|
||||||
user_id: sessionUser.id,
|
|
||||||
stash_id: stashId,
|
|
||||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return fetchStashes('movie', movieId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function unstashActor(actorId, stashId, sessionUser) {
|
|
||||||
await knex
|
|
||||||
.from('stashes_actors AS deletable')
|
|
||||||
.where('deletable.actor_id', actorId)
|
|
||||||
.where('deletable.stash_id', stashId)
|
|
||||||
.whereExists(knex('stashes_actors') // verify user owns this stash, complimentary to row-level security
|
|
||||||
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
|
|
||||||
.where('stashes_actors.stash_id', knex.raw('deletable.stash_id'))
|
|
||||||
.where('stashes.user_id', sessionUser.id))
|
|
||||||
.delete();
|
|
||||||
|
|
||||||
await indexApi.callDelete({
|
|
||||||
index: 'actors_stashed',
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{ equals: { actor_id: actorId } },
|
|
||||||
{ equals: { stash_id: stashId } },
|
|
||||||
{ equals: { user_id: sessionUser.id } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
refreshActorsView();
|
|
||||||
|
|
||||||
return fetchStashes('actor', actorId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function unstashScene(sceneId, stashId, sessionUser) {
|
|
||||||
await knex
|
|
||||||
.from('stashes_scenes AS deletable')
|
|
||||||
.where('deletable.scene_id', sceneId)
|
|
||||||
.where('deletable.stash_id', stashId)
|
|
||||||
.whereExists(knex('stashes_scenes') // verify user owns this stash, complimentary to row-level security
|
|
||||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
|
||||||
.where('stashes_scenes.stash_id', knex.raw('deletable.stash_id'))
|
|
||||||
.where('stashes.user_id', sessionUser.id))
|
|
||||||
.delete();
|
|
||||||
|
|
||||||
await indexApi.callDelete({
|
|
||||||
index: 'scenes_stashed',
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{ equals: { scene_id: sceneId } },
|
|
||||||
{ equals: { stash_id: stashId } },
|
|
||||||
{ equals: { user_id: sessionUser.id } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return fetchStashes('scene', sceneId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function unstashMovie(movieId, stashId, sessionUser) {
|
|
||||||
await knex
|
|
||||||
.from('stashes_movies AS deletable')
|
|
||||||
.where('deletable.movie_id', movieId)
|
|
||||||
.where('deletable.stash_id', stashId)
|
|
||||||
.whereExists(knex('stashes_movies') // verify user owns this stash, complimentary to row-level security
|
|
||||||
.leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id')
|
|
||||||
.where('stashes_movies.stash_id', knex.raw('deletable.stash_id'))
|
|
||||||
.where('stashes.user_id', sessionUser.id))
|
|
||||||
.delete();
|
|
||||||
|
|
||||||
await indexApi.callDelete({
|
|
||||||
index: 'movies_stashed',
|
|
||||||
query: {
|
|
||||||
bool: {
|
|
||||||
must: [
|
|
||||||
{ equals: { movie_id: movieId } },
|
|
||||||
{ equals: { stash_id: stashId } },
|
|
||||||
{ equals: { user_id: sessionUser.id } },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return fetchStashes('movie', movieId, sessionUser);
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
createStash,
|
|
||||||
curateStash,
|
|
||||||
removeStash,
|
|
||||||
stashActor,
|
|
||||||
stashScene,
|
|
||||||
stashMovie,
|
|
||||||
unstashScene,
|
|
||||||
unstashActor,
|
|
||||||
unstashMovie,
|
|
||||||
updateStash,
|
|
||||||
};
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const { decode } = require('html-entities');
|
const { decode: decodeEntities } = require('html-entities');
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
|
|
||||||
const argv = require('./argv');
|
const argv = require('./argv');
|
||||||
@@ -28,6 +28,11 @@ const commonContext = {
|
|||||||
unprint,
|
unprint,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// newer html-entities throws on non-strings, pass e.g. numeric titles through like 2.4 did
|
||||||
|
function decode(text) {
|
||||||
|
return typeof text === 'string' ? decodeEntities(text) : text;
|
||||||
|
}
|
||||||
|
|
||||||
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
||||||
const slugBase = release.title
|
const slugBase = release.title
|
||||||
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const fetch = require('node-fetch');
|
|
||||||
const express = require('express');
|
|
||||||
|
|
||||||
async function init() {
|
|
||||||
const res = await fetch('https://www.realitykings.com/scenes?site=45', {
|
|
||||||
method: 'HEAD',
|
|
||||||
headers: {
|
|
||||||
'user-agent': 'HTTPie/2.6.0',
|
|
||||||
'accept-encoding': 'gzip, deflate, br',
|
|
||||||
accept: '*/*',
|
|
||||||
connection: 'keep-alive',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(res.status, res.headers);
|
|
||||||
|
|
||||||
const app = express();
|
|
||||||
|
|
||||||
app.get('/', (appReq, appRes) => {
|
|
||||||
console.log(appReq.headers);
|
|
||||||
appRes.status(204).send();
|
|
||||||
});
|
|
||||||
|
|
||||||
app.listen(8000, () => {
|
|
||||||
console.log('Listening on port 8000');
|
|
||||||
|
|
||||||
fetch('http://127.0.0.1:8000', {
|
|
||||||
headers: {
|
|
||||||
'user-agent': 'HTTPie/2.6.0',
|
|
||||||
'accept-encoding': 'gzip, deflate, br',
|
|
||||||
accept: '*/*',
|
|
||||||
connection: 'keep-alive',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
init();
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const express = require('express');
|
|
||||||
|
|
||||||
const app = express();
|
|
||||||
|
|
||||||
app.get('/', (req, res) => {
|
|
||||||
console.log(req.httpVersion);
|
|
||||||
console.log(req.headers);
|
|
||||||
console.log(req.body);
|
|
||||||
|
|
||||||
res.status(204).send();
|
|
||||||
});
|
|
||||||
|
|
||||||
app.listen(5555, () => {
|
|
||||||
console.log('Server started');
|
|
||||||
});
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
// const util = require('util');
|
// const util = require('util');
|
||||||
const unprint = require('unprint');
|
const unprint = require('unprint');
|
||||||
const args = require('yargs').argv;
|
const args = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
|
||||||
|
|
||||||
const knex = require('../knex');
|
const knex = require('../knex');
|
||||||
const bulkInsert = require('../utils/bulk-insert');
|
const bulkInsert = require('../utils/bulk-insert');
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const knex = require('./knex');
|
|
||||||
const { curateStash } = require('./stashes');
|
|
||||||
|
|
||||||
function curateUser(user) {
|
|
||||||
if (!user) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const ability = [...(user.role_abilities || []), ...(user.abilities || [])];
|
|
||||||
|
|
||||||
const curatedUser = {
|
|
||||||
id: user.id,
|
|
||||||
username: user.username,
|
|
||||||
email: user.email,
|
|
||||||
emailVerified: user.email_verified,
|
|
||||||
identityVerified: user.identity_verified,
|
|
||||||
ability,
|
|
||||||
createdAt: user.created_at,
|
|
||||||
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
|
|
||||||
};
|
|
||||||
|
|
||||||
return curatedUser;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchUser(userId, raw) {
|
|
||||||
const user = await knex('users')
|
|
||||||
.select(knex.raw('users.*, users_roles.abilities as role_abilities, COALESCE(json_agg(stashes ORDER BY stashes.created_at) FILTER (WHERE stashes.id IS NOT NULL), \'[]\') as stashes'))
|
|
||||||
.modify((builder) => {
|
|
||||||
if (typeof userId === 'number') {
|
|
||||||
builder.where('users.id', userId);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof userId === 'string') {
|
|
||||||
builder
|
|
||||||
.where('users.username', userId)
|
|
||||||
.orWhere('users.email', userId);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.leftJoin('users_roles', 'users_roles.role', 'users.role')
|
|
||||||
.leftJoin('stashes', 'stashes.user_id', 'users.id')
|
|
||||||
.groupBy('users.id', 'users_roles.role')
|
|
||||||
.first();
|
|
||||||
|
|
||||||
if (raw) {
|
|
||||||
return user;
|
|
||||||
}
|
|
||||||
|
|
||||||
return curateUser(user);
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
curateUser,
|
|
||||||
fetchUser,
|
|
||||||
};
|
|
||||||
+1
-1
@@ -6,7 +6,7 @@ const fsPromises = require('fs').promises;
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const { PassThrough } = require('stream');
|
const { PassThrough } = require('stream');
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const argv = require('yargs').argv;
|
const argv = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
|
||||||
|
|
||||||
const file = 'https://speed.hetzner.de/100MB.bin';
|
const file = 'https://speed.hetzner.de/100MB.bin';
|
||||||
// const file = 'https://speed.hetzner.de/1GB.bin';
|
// const file = 'https://speed.hetzner.de/1GB.bin';
|
||||||
|
|||||||
-153
@@ -1,153 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
const arrayEqual = require('array-equal');
|
|
||||||
|
|
||||||
/* eslint-disable */
|
|
||||||
function evaluateBranch(tree, modifiers) {
|
|
||||||
const result = tree.map((expression) => {
|
|
||||||
if (expression.type === 'group') {
|
|
||||||
return evaluateBranch(expression.values, expression.modifiers);
|
|
||||||
} if (expression.modifiers.length === 0) {
|
|
||||||
return ''; // This is a trigger to stringify the previous values
|
|
||||||
} if (arrayEqual(['plus'], expression.modifiers)) {
|
|
||||||
return 0;
|
|
||||||
} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
|
|
||||||
return true;
|
|
||||||
} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
|
|
||||||
return true;
|
|
||||||
} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
|
|
||||||
return true;
|
|
||||||
} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
|
|
||||||
}).reduce((combined, value) => {
|
|
||||||
if (value === '') {
|
|
||||||
return combined.toString();
|
|
||||||
}
|
|
||||||
if (value === true) {
|
|
||||||
value = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof combined === 'string') {
|
|
||||||
return combined + value.toString();
|
|
||||||
}
|
|
||||||
return combined + value;
|
|
||||||
}, 0);
|
|
||||||
|
|
||||||
if (modifiers == null) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
if (arrayEqual(['plus'], modifiers)) {
|
|
||||||
return parseInt(result);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
function evaluate(tree) {
|
|
||||||
return evaluateBranch(tree);
|
|
||||||
}
|
|
||||||
|
|
||||||
function parse(string) {
|
|
||||||
const length = string.length;
|
|
||||||
let byte;
|
|
||||||
|
|
||||||
let stateFinishedItem = false;
|
|
||||||
const modifierStack = [[]];
|
|
||||||
const itemStack = [[]];
|
|
||||||
let currentStack = itemStack[0];
|
|
||||||
let currentModifiers = modifierStack[0];
|
|
||||||
let stackLevel = 0;
|
|
||||||
|
|
||||||
for (let pos = 0; pos < length; pos++) {
|
|
||||||
byte = string[pos];
|
|
||||||
|
|
||||||
switch (byte) {
|
|
||||||
case '+':
|
|
||||||
if (pos === 0 || stateFinishedItem === false) {
|
|
||||||
// Modifier / number-cast
|
|
||||||
currentModifiers.push('plus');
|
|
||||||
stateFinishedItem = false;
|
|
||||||
} else {
|
|
||||||
// Addition, we don't need to do anything here
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case '!':
|
|
||||||
stateFinishedItem = false;
|
|
||||||
currentModifiers.push('negate');
|
|
||||||
break;
|
|
||||||
case '(':
|
|
||||||
stateFinishedItem = false;
|
|
||||||
stackLevel++;
|
|
||||||
itemStack[stackLevel] = currentStack = [];
|
|
||||||
modifierStack[stackLevel] = currentModifiers = [];
|
|
||||||
break;
|
|
||||||
case ')':
|
|
||||||
if (stackLevel === 0) {
|
|
||||||
throw new Error('Encountered ) without matching (');
|
|
||||||
}
|
|
||||||
|
|
||||||
stackLevel--;
|
|
||||||
stateFinishedItem = true;
|
|
||||||
|
|
||||||
currentStack = itemStack[stackLevel];
|
|
||||||
currentStack.push({
|
|
||||||
type: 'group',
|
|
||||||
values: itemStack[stackLevel + 1],
|
|
||||||
modifiers: modifierStack[stackLevel],
|
|
||||||
});
|
|
||||||
|
|
||||||
currentModifiers = modifierStack[stackLevel] = [];
|
|
||||||
break;
|
|
||||||
case '[':
|
|
||||||
if (string[pos + 1] === ']') {
|
|
||||||
// Reached the brackets; end of the modifier sequence
|
|
||||||
currentStack.push({
|
|
||||||
type: 'brackets',
|
|
||||||
modifiers: currentModifiers,
|
|
||||||
});
|
|
||||||
|
|
||||||
currentModifiers = [];
|
|
||||||
pos += 1; // Skip over the closing bracket
|
|
||||||
stateFinishedItem = true;
|
|
||||||
} else {
|
|
||||||
throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return itemStack[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseExpression(string) {
|
|
||||||
return evaluate(parse(string));
|
|
||||||
}
|
|
||||||
|
|
||||||
function findAll(regex, target) {
|
|
||||||
const results = []; let
|
|
||||||
match;
|
|
||||||
|
|
||||||
while (match = regex.exec(target)) {
|
|
||||||
results.push(match);
|
|
||||||
}
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
function cf(testcase) {
|
|
||||||
let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
|
|
||||||
|
|
||||||
const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
|
|
||||||
operation: match[1],
|
|
||||||
expression: match[2],
|
|
||||||
})).map(({ operation, expression }) => ({
|
|
||||||
operation,
|
|
||||||
expression: parseExpression(expression),
|
|
||||||
}));
|
|
||||||
|
|
||||||
initialExpression = parseExpression(initialExpression);
|
|
||||||
|
|
||||||
return { parent, child, initialExpression, modifyingExpressions };
|
|
||||||
};
|
|
||||||
|
|
||||||
module.exports = cf;
|
|
||||||
@@ -82,7 +82,7 @@ function convertApi(rawInput, fromOrTo, to) {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const input = decode(rawInput); // remove html entities, e.g. 5' 8" for 5' 8"
|
const input = typeof rawInput === 'string' ? decode(rawInput) : rawInput; // remove html entities, e.g. 5' 8" for 5' 8"
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (typeof input === 'string' && to === undefined) {
|
if (typeof input === 'string' && to === undefined) {
|
||||||
|
|||||||
+3
-269
@@ -2,16 +2,13 @@
|
|||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const bhttp = require('bhttp');
|
|
||||||
const undici = require('undici');
|
const undici = require('undici');
|
||||||
const fs = require('fs').promises;
|
const fs = require('fs').promises;
|
||||||
// const util = require('util');
|
// const util = require('util');
|
||||||
// const stream = require('stream');
|
// const stream = require('stream');
|
||||||
const { pipeline } = require('stream/promises');
|
const { pipeline } = require('stream/promises');
|
||||||
const Bottleneck = require('bottleneck');
|
const Bottleneck = require('bottleneck');
|
||||||
const { JSDOM, toughCookie } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const puppeteer = require('puppeteer-extra');
|
|
||||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
|
||||||
|
|
||||||
const windows = require('./http-windows');
|
const windows = require('./http-windows');
|
||||||
|
|
||||||
@@ -21,24 +18,12 @@ const argv = require('../argv');
|
|||||||
|
|
||||||
// const pipeline = util.promisify(stream.pipeline);
|
// const pipeline = util.promisify(stream.pipeline);
|
||||||
|
|
||||||
const limiters = {
|
const limiters = {};
|
||||||
bypass: new Bottleneck({
|
|
||||||
minTime: 1000,
|
|
||||||
maxConcurrent: 1,
|
|
||||||
timeout: 60000,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
|
|
||||||
const bypassSessions = new Map();
|
|
||||||
|
|
||||||
const browsers = new Map();
|
|
||||||
|
|
||||||
Promise.config({
|
Promise.config({
|
||||||
cancellation: true,
|
cancellation: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
puppeteer.use(StealthPlugin());
|
|
||||||
|
|
||||||
const defaultOptions = {
|
const defaultOptions = {
|
||||||
timeout: argv.requestTimeout,
|
timeout: argv.requestTimeout,
|
||||||
encodeJSON: true,
|
encodeJSON: true,
|
||||||
@@ -58,50 +43,6 @@ function useProxy(url) {
|
|||||||
return config.proxy.hostnames.includes(hostname);
|
return config.proxy.hostnames.includes(hostname);
|
||||||
}
|
}
|
||||||
|
|
||||||
function useBrowserBypass(url, options) {
|
|
||||||
if (!config.bypass.browser.enable) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { hostname } = new URL(url);
|
|
||||||
|
|
||||||
if (options.bypassBrowser === 'shared') {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.bypass.browser.hostnames.includes(hostname)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function useCloudflareBypass(url, options) {
|
|
||||||
if (!config.bypass.cloudflare.enable) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { hostname } = new URL(url);
|
|
||||||
|
|
||||||
if (options.bypassCloudflare === 'shared') {
|
|
||||||
return 'shared';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.bypassCloudflare === 'independent') {
|
|
||||||
return hostname;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
|
|
||||||
return 'shared';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
|
|
||||||
return hostname;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const propMap = {
|
const propMap = {
|
||||||
media: {
|
media: {
|
||||||
interval: 'mediaInterval',
|
interval: 'mediaInterval',
|
||||||
@@ -146,180 +87,6 @@ function getLimiter(options = {}, url) {
|
|||||||
return limiters[interval][concurrency];
|
return limiters[interval][concurrency];
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractJson(body, headers) {
|
|
||||||
if (headers['content-type'].includes('application/json')) {
|
|
||||||
const { document } = new JSDOM(body, { virtualConsole }).window;
|
|
||||||
const dataString = document.querySelector('body > pre')?.textContent;
|
|
||||||
|
|
||||||
if (dataString) {
|
|
||||||
const data = JSON.parse(dataString);
|
|
||||||
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getBrowserSession(identifier, options = {}) {
|
|
||||||
return limiters.bypass.schedule(async () => {
|
|
||||||
const browserIdentifier = options.useGlobalBrowser === false ? identifier : null;
|
|
||||||
|
|
||||||
if (!browsers.has(browserIdentifier)) {
|
|
||||||
const newBrowser = await puppeteer.launch({
|
|
||||||
headless: typeof options.headless === 'undefined' ? 'new' : options.headless,
|
|
||||||
args: [
|
|
||||||
...(options.useProxy && config.proxy.enabled ? [`--proxy-server=${config.proxy.host}:${config.proxy.port}`] : []),
|
|
||||||
],
|
|
||||||
// headless: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
browsers.set(browserIdentifier, newBrowser);
|
|
||||||
|
|
||||||
logger.info('Initialized puppeteer browser');
|
|
||||||
}
|
|
||||||
|
|
||||||
const browser = browsers.get(browserIdentifier);
|
|
||||||
const tab = await browser.newPage();
|
|
||||||
|
|
||||||
logger.verbose(`Opened puppeteer tab${identifier ? ` for ${identifier}` : ''}`);
|
|
||||||
|
|
||||||
return { browser, tab };
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function bypassBrowserRequest(url, options) {
|
|
||||||
const { tab } = await getBrowserSession(new URL(url).hostname, options);
|
|
||||||
|
|
||||||
const res = await tab.goto(url);
|
|
||||||
|
|
||||||
if (options.bypass?.delay) {
|
|
||||||
await Promise.delay(options.bypass.delay);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof options.bypass?.evaluate === 'function') {
|
|
||||||
await tab.evaluate(options.bypass.evaluate, options.bypass);
|
|
||||||
}
|
|
||||||
|
|
||||||
const rawBody = await tab.content();
|
|
||||||
|
|
||||||
const headers = res.headers();
|
|
||||||
const body = extractJson(rawBody, headers);
|
|
||||||
|
|
||||||
const statusCode = res.status();
|
|
||||||
|
|
||||||
if (!statusCode === 200) {
|
|
||||||
throw new Error(`Puppeteer bypass failed for ${url} (${statusCode}): ${body?.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
body,
|
|
||||||
statusCode,
|
|
||||||
headers,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getBypassSession(url, hostname) {
|
|
||||||
if (bypassSessions.has(hostname)) {
|
|
||||||
return bypassSessions.get(hostname);
|
|
||||||
}
|
|
||||||
|
|
||||||
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
|
|
||||||
cmd: 'sessions.create',
|
|
||||||
proxy: useProxy(url) ? {
|
|
||||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
|
||||||
} : null,
|
|
||||||
}, {
|
|
||||||
encodeJSON: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
|
|
||||||
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
bypassSessions.set(hostname, sessionRes.body.session);
|
|
||||||
|
|
||||||
return sessionRes.body.session;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function destroyBypassSession(sessionId) {
|
|
||||||
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
|
||||||
cmd: 'sessions.destroy',
|
|
||||||
session: sessionId,
|
|
||||||
}, {
|
|
||||||
encodeJSON: true,
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
|
|
||||||
bypassSessions.delete(sessionId);
|
|
||||||
|
|
||||||
logger.verbose(`Destroyed bypass session ${sessionId}`);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function destroyBypassSessions() {
|
|
||||||
if (!config.bypass.cloudflare.enabled) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
|
||||||
cmd: 'sessions.list',
|
|
||||||
}, {
|
|
||||||
encodeJSON: true,
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
|
|
||||||
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
|
|
||||||
}
|
|
||||||
|
|
||||||
async function destroyBrowserSessions() {
|
|
||||||
await Promise.all(Array.from(browsers.values(), async (browser) => browser?.close()));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
|
|
||||||
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
|
|
||||||
|
|
||||||
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
|
|
||||||
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
|
||||||
cmd: `request.${method}`,
|
|
||||||
url,
|
|
||||||
session: sessionId,
|
|
||||||
maxTimeout: options.timeout,
|
|
||||||
proxy: useProxy(url) ? {
|
|
||||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
|
||||||
} : null,
|
|
||||||
}, {
|
|
||||||
encodeJSON: true,
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
|
|
||||||
if (/session closed/i.test(res.body?.message) && attempts < 3) {
|
|
||||||
await destroyBypassSession(sessionId);
|
|
||||||
|
|
||||||
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const resBody = extractJson(res.body.solution.response, res.body.solution.headers);
|
|
||||||
|
|
||||||
return {
|
|
||||||
body: resBody,
|
|
||||||
statusCode: res.body.solution.status,
|
|
||||||
headers: res.body.solution.headers,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const defaultAgent = new undici.Agent({
|
const defaultAgent = new undici.Agent({
|
||||||
allowH2: true,
|
allowH2: true,
|
||||||
connect: {
|
connect: {
|
||||||
@@ -333,19 +100,8 @@ const proxyAgent = new undici.ProxyAgent({
|
|||||||
|
|
||||||
async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
|
async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
|
||||||
const withProxy = useProxy(url);
|
const withProxy = useProxy(url);
|
||||||
const withBrowserBypass = useBrowserBypass(url, requestOptions);
|
|
||||||
const withCloudflareBypass = useCloudflareBypass(url, requestOptions);
|
|
||||||
|
|
||||||
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
|
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
|
||||||
|
|
||||||
if (withBrowserBypass) {
|
|
||||||
if (method !== 'get') throw new Error('Browser bypass only supports GET');
|
|
||||||
return bypassBrowserRequest(url, requestOptions);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (withCloudflareBypass) {
|
|
||||||
return bypassCloudflareRequest(url, method, body, withCloudflareBypass, requestOptions);
|
|
||||||
}
|
|
||||||
|
|
||||||
const headers = {
|
const headers = {
|
||||||
...requestOptions.headers,
|
...requestOptions.headers,
|
||||||
@@ -479,33 +235,11 @@ async function head(url, options) {
|
|||||||
return scheduleRequest('head', url, null, options);
|
return scheduleRequest('head', url, null, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getSession(options) {
|
|
||||||
return bhttp.session({ ...defaultOptions, ...options });
|
|
||||||
}
|
|
||||||
|
|
||||||
function getCookieJar(store, options) {
|
|
||||||
return new toughCookie.CookieJar(store, {
|
|
||||||
looseMode: true,
|
|
||||||
...options,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
toughCookie,
|
|
||||||
get,
|
get,
|
||||||
head,
|
head,
|
||||||
post,
|
post,
|
||||||
delete: del,
|
delete: del,
|
||||||
put,
|
put,
|
||||||
patch,
|
patch,
|
||||||
session: getSession,
|
|
||||||
cookieJar: getCookieJar,
|
|
||||||
browsers,
|
|
||||||
getBrowserSession,
|
|
||||||
getBypassSession,
|
|
||||||
getSession,
|
|
||||||
getCookieJar,
|
|
||||||
destroyBypassSessions,
|
|
||||||
destroyBrowserSessions,
|
|
||||||
proxyAgent,
|
|
||||||
};
|
};
|
||||||
|
|||||||
+1
-1
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
const bhttp = require('bhttp');
|
const bhttp = require('bhttp');
|
||||||
const sharp = require('sharp');
|
const sharp = require('sharp');
|
||||||
const { argv } = require('yargs');
|
const { argv } = require('yargs')(require('yargs/helpers').hideBin(process.argv));
|
||||||
|
|
||||||
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
|
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
|
||||||
|
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
const webpack = require('webpack');
|
|
||||||
const path = require('path');
|
|
||||||
const { VueLoaderPlugin } = require('vue-loader');
|
|
||||||
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
|
|
||||||
const autoprefixer = require('autoprefixer');
|
|
||||||
const ESLintPlugin = require('eslint-webpack-plugin');
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
entry: {
|
|
||||||
bundle: './assets/js/main.js',
|
|
||||||
},
|
|
||||||
output: {
|
|
||||||
filename: '[name].js',
|
|
||||||
path: path.join(__dirname, 'public/js'),
|
|
||||||
},
|
|
||||||
module: {
|
|
||||||
rules: [
|
|
||||||
{
|
|
||||||
test: /\.vue$/,
|
|
||||||
include: [
|
|
||||||
path.resolve(__dirname, 'assets'),
|
|
||||||
],
|
|
||||||
loader: 'vue-loader',
|
|
||||||
options: {
|
|
||||||
preserveWhitespace: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
test: /\.js$/,
|
|
||||||
exclude: /node_modules/,
|
|
||||||
use: [
|
|
||||||
{
|
|
||||||
loader: 'babel-loader',
|
|
||||||
options: {
|
|
||||||
babelrc: false,
|
|
||||||
plugins: [
|
|
||||||
'@babel/plugin-proposal-object-rest-spread',
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
test: /\.s?css$/,
|
|
||||||
use: [
|
|
||||||
MiniCssExtractPlugin.loader,
|
|
||||||
{
|
|
||||||
loader: 'css-loader',
|
|
||||||
options: {
|
|
||||||
sourceMap: true,
|
|
||||||
url: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
loader: 'postcss-loader',
|
|
||||||
options: {
|
|
||||||
postcssOptions: {
|
|
||||||
plugins: [autoprefixer],
|
|
||||||
},
|
|
||||||
sourceMap: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
loader: 'sass-loader',
|
|
||||||
options: {
|
|
||||||
sourceMap: true,
|
|
||||||
additionalData: '@import "assets/css/_breakpoints.scss";',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
test: /\.svg/,
|
|
||||||
use: 'raw-loader',
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
plugins: [
|
|
||||||
new VueLoaderPlugin(),
|
|
||||||
new ESLintPlugin(),
|
|
||||||
new MiniCssExtractPlugin({
|
|
||||||
filename: '../css/style.css',
|
|
||||||
}),
|
|
||||||
new webpack.DefinePlugin({
|
|
||||||
VERSION: JSON.stringify(process.env.npm_package_version),
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
resolve: {
|
|
||||||
alias: {
|
|
||||||
theme: path.join(__dirname, 'assets/css/_theme.scss'),
|
|
||||||
breakpoints: path.join(__dirname, 'assets/css/_breakpoints.scss'),
|
|
||||||
config: path.join(__dirname, `assets/js/config/${process.env.NODE_ENV || 'default'}.js`),
|
|
||||||
vue: 'vue/dist/vue.esm-bundler.js',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
Reference in New Issue
Block a user