Updated dependencies not locked by require.
This commit is contained in:
DebaucheryLibrarian
+3
-3
@@ -4,11 +4,11 @@ const config = require('config');
|
||||
const util = require('util');
|
||||
const Promise = require('bluebird');
|
||||
const moment = require('moment');
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
const DOMPurify = require('dompurify');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const omit = require('object.omit');
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
const unprint = require('unprint');
|
||||
|
||||
const { window } = new JSDOM('');
|
||||
@@ -413,7 +413,7 @@ async function curateProfile(profile, actor) {
|
||||
|
||||
curatedProfile.description = domPurify.sanitize(profile.description?.replace(/\s+/g, ' '), { ALLOWED_TAGS: [] }).trim() || null;
|
||||
|
||||
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(slugify(curatedProfile.description), { dkLen: 24 })).toString('hex');
|
||||
curatedProfile.descriptionHash = curatedProfile.description && Buffer.from(blake2b(Buffer.from(slugify(curatedProfile.description)), { dkLen: 24 })).toString('hex');
|
||||
|
||||
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
|
||||
|
||||
|
||||
+2
-4
@@ -5,7 +5,7 @@ const startTime = new Date();
|
||||
const config = require('config');
|
||||
const util = require('util');
|
||||
const unprint = require('unprint');
|
||||
const log = require('why-is-node-running');
|
||||
const log = require('why-is-node-running').default;
|
||||
const Inspector = require('inspector-api');
|
||||
const fs = require('fs').promises;
|
||||
const { format, intervalToDuration } = require('date-fns');
|
||||
@@ -269,12 +269,10 @@ async function init() {
|
||||
logger.error(error);
|
||||
}
|
||||
|
||||
await http.destroyBypassSessions();
|
||||
await http.destroyBrowserSessions();
|
||||
await unprint.closeAllBrowsers();
|
||||
|
||||
knex.destroy();
|
||||
redis.disconnect();
|
||||
redis.destroy();
|
||||
|
||||
done = true;
|
||||
}
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
|
||||
const config = require('config');
|
||||
const yargs = require('yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
const moment = require('moment');
|
||||
|
||||
function interpretDate(after, ignoreIfEmpty = false) {
|
||||
@@ -27,7 +28,7 @@ function interpretDate(after, ignoreIfEmpty = false) {
|
||||
.toDate();
|
||||
}
|
||||
|
||||
const { argv } = yargs
|
||||
const { argv } = yargs(hideBin(process.argv))
|
||||
.command('npm start')
|
||||
.command('update', 'fetch latest updates', () => {
|
||||
console.log('UPDATING!');
|
||||
|
||||
-110
@@ -1,110 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const util = require('util');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const knex = require('./knex');
|
||||
const { curateUser, fetchUser } = require('./users');
|
||||
const { HttpError } = require('./errors');
|
||||
|
||||
const scrypt = util.promisify(crypto.scrypt);
|
||||
|
||||
async function verifyPassword(password, storedPassword) {
|
||||
const [salt, hash] = storedPassword.split('/');
|
||||
const hashedPassword = (await scrypt(password, salt, 64)).toString('hex');
|
||||
|
||||
if (hashedPassword === hash) {
|
||||
return true;
|
||||
}
|
||||
|
||||
throw new HttpError('Username or password incorrect', 401);
|
||||
}
|
||||
|
||||
async function login(credentials) {
|
||||
if (!config.auth.login) {
|
||||
throw new HttpError('Authentication is disabled', 405);
|
||||
}
|
||||
|
||||
const user = await fetchUser(credentials.username.trim(), true);
|
||||
|
||||
if (!user) {
|
||||
throw new HttpError('Username or password incorrect', 401);
|
||||
}
|
||||
|
||||
await verifyPassword(credentials.password, user.password);
|
||||
|
||||
await knex('users')
|
||||
.update('last_login', 'NOW()')
|
||||
.where('id', user.id);
|
||||
|
||||
return curateUser(user);
|
||||
}
|
||||
|
||||
async function signup(credentials) {
|
||||
if (!config.auth.signup) {
|
||||
throw new HttpError('Authentication is disabled', 405);
|
||||
}
|
||||
|
||||
const curatedUsername = credentials.username.trim();
|
||||
|
||||
if (!curatedUsername) {
|
||||
throw new HttpError('Username required', 400);
|
||||
}
|
||||
|
||||
if (curatedUsername.length < config.auth.usernameLength[0]) {
|
||||
throw new HttpError('Username is too short', 400);
|
||||
}
|
||||
|
||||
if (curatedUsername.length > config.auth.usernameLength[1]) {
|
||||
throw new HttpError('Username is too long', 400);
|
||||
}
|
||||
|
||||
if (!config.auth.usernamePattern.test(curatedUsername)) {
|
||||
throw new HttpError('Username contains invalid characters', 400);
|
||||
}
|
||||
|
||||
if (!credentials.email) {
|
||||
throw new HttpError('E-mail required', 400);
|
||||
}
|
||||
|
||||
if (credentials.password?.length < 3) {
|
||||
throw new HttpError('Password must be 3 characters or longer', 400);
|
||||
}
|
||||
|
||||
const existingUser = await knex('users')
|
||||
.where('username', curatedUsername)
|
||||
.orWhere('email', credentials.email)
|
||||
.first();
|
||||
|
||||
if (existingUser) {
|
||||
throw new HttpError('Username or e-mail already in use', 409);
|
||||
}
|
||||
|
||||
const salt = crypto.randomBytes(16).toString('hex');
|
||||
const hashedPassword = (await scrypt(credentials.password, salt, 64)).toString('hex');
|
||||
const storedPassword = `${salt}/${hashedPassword}`;
|
||||
|
||||
const [{ id: userId }] = await knex('users')
|
||||
.insert({
|
||||
username: curatedUsername,
|
||||
email: credentials.email,
|
||||
password: storedPassword,
|
||||
})
|
||||
.returning('id');
|
||||
|
||||
await knex('stashes').insert({
|
||||
user_id: userId,
|
||||
name: 'Favorites',
|
||||
slug: 'favorites',
|
||||
public: false,
|
||||
primary: true,
|
||||
});
|
||||
|
||||
return fetchUser(userId);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
login,
|
||||
signup,
|
||||
};
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const argv = require('./argv');
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
require('babel-polyfill');
|
||||
const init = require('./app');
|
||||
|
||||
init();
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
module.exports = {
|
||||
searchApi: new manticore.SearchApi(mantiClient),
|
||||
indexApi: new manticore.IndexApi(mantiClient),
|
||||
utilsApi: new manticore.UtilsApi(),
|
||||
};
|
||||
+2
-2
@@ -8,11 +8,11 @@ const fsPromises = require('fs').promises;
|
||||
const path = require('path');
|
||||
const stream = require('stream');
|
||||
const { nanoid } = require('nanoid/non-secure');
|
||||
const mime = require('mime');
|
||||
const mime = require('mime').default;
|
||||
// const fileType = require('file-type');
|
||||
const ffmpeg = require('fluent-ffmpeg');
|
||||
const sharp = require('sharp');
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
const { format } = require('date-fns');
|
||||
const taskQueue = require('promise-task-queue');
|
||||
const unprint = require('unprint');
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
const inquirer = require('inquirer');
|
||||
const inquirer = require('inquirer').default;
|
||||
|
||||
const logger = require('./logger')(__filename);
|
||||
const knex = require('./knex');
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
const { blake2b } = require('@noble/hashes/blake2b');
|
||||
const { blake2b } = require('@noble/hashes/blake2.js');
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
function getHash(identifier) {
|
||||
return Buffer.from(blake2b(identifier, { dkLen: 8 })).toString('hex');
|
||||
return Buffer.from(blake2b(Buffer.from(identifier), { dkLen: 8 })).toString('hex');
|
||||
}
|
||||
|
||||
function extractMaleModelsFromTags(tagContainer) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const mime = require('mime');
|
||||
const mime = require('mime').default;
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const escapeRegexp = require('escape-string-regexp');
|
||||
const escapeRegexp = require('escape-string-regexp').default;
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const capitalize = require('../utils/capitalize');
|
||||
|
||||
+14
-29
@@ -47,11 +47,7 @@ function scrapeAll(scenes, tilesByEntryId, channel) {
|
||||
|
||||
release.tags = tagsMap[query.content('a[href*="type/"]')?.toLowerCase()];
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
|
||||
if (posterBackground?.background) {
|
||||
release.poster = posterBackground.background.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
}
|
||||
release.poster = query.background('.player');
|
||||
|
||||
release.photos = query.all('.sample-thumbs .thumb a').map((linkEl) => [
|
||||
unprint.query.url(linkEl, null),
|
||||
@@ -121,8 +117,7 @@ function scrapeAllTiles(tiles, channel) {
|
||||
|
||||
release.tags = [...query.contents('.data a[href*="/tag"]'), ...(tagsMap[query.content('.scene-type')?.toLowerCase()] || [])].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.scene-img');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.scene-img');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -199,8 +194,7 @@ function scrapeAllCospuri(scenes, channel) {
|
||||
|
||||
release.tags = [...query.contents('.tags .tag, .tag-box .tag'), query.content('.model .channel')].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.scene-thumb');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.scene-thumb');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -260,8 +254,7 @@ function scrapeAllFellatio(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('.tour-data a[href*="tag/"]');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -302,8 +295,7 @@ function scrapeAllHandjob(scenes, _channel) {
|
||||
|
||||
release.actors = query.text('.item-ltitle h1')?.split(/,\s*/).map((actor) => actor.trim());
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -348,8 +340,7 @@ function scrapeAllLegs(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('a[href*="tag/"]');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -385,8 +376,7 @@ function scrapeAllFacefuck(scenes) {
|
||||
release.description = query.content('.infotxt');
|
||||
release.actors = query.content('.info h1').split(',').map((actor) => actor.trim());
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -425,8 +415,7 @@ function scrapeAllTrans(scenes) {
|
||||
release.duration = query.duration('.sample-info');
|
||||
release.photoCount = albumContext?.query.number('.sample-info', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -436,7 +425,7 @@ function scrapeAllTrans(scenes) {
|
||||
];
|
||||
}
|
||||
|
||||
release.photos = curatePhotos(albumContext?.query.styles('.sample-lg, .sample-thumb').map((style) => style['background-image']?.match(/url\((.*)\)/)?.[1]));
|
||||
release.photos = curatePhotos(albumContext?.query.backgrounds('.sample-lg, .sample-thumb'));
|
||||
release.trailer = query.video();
|
||||
|
||||
release.entryId = entryIdFromMedia(release);
|
||||
@@ -486,8 +475,7 @@ function scrapeAllLesbianTiles(scenes, channel) {
|
||||
|
||||
release.tags = query.contents('.content-tags a');
|
||||
|
||||
const posterBackground = query.style('.vidthumb');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.vidthumb');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -522,8 +510,7 @@ function scrapeAllLesbian(scenes, channel, tiles) {
|
||||
release.videoCount = query.number('.tour-datum', { match: /(\d+) hd scenes/i, matchIndex: 1 });
|
||||
release.photoCount = query.number('//div[text()[contains(., "Photos")]]', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -650,8 +637,7 @@ function scrapeSceneBuffet({ query }, { url, entity }) {
|
||||
|
||||
release.tags = query.contents('.tag-list a');
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = [
|
||||
@@ -696,8 +682,7 @@ function scrapeSceneCospuri({ query }, { url, entity }) {
|
||||
|
||||
release.tags = [...query.contents('.tag'), query.content('.sample-channel')].filter(Boolean);
|
||||
|
||||
const posterBackground = query.style('.player');
|
||||
const posterUrl = posterBackground?.background?.match(/url\((.*)\)/)?.[1]?.trim();
|
||||
const posterUrl = query.background('.player');
|
||||
|
||||
if (posterUrl) {
|
||||
release.poster = posterUrl;
|
||||
@@ -788,7 +773,7 @@ function scrapeProfile({ query }, _channel, url) {
|
||||
|
||||
const avatar = query.img('.scene-array img[src*="/actress"], img.portrait, .profile-img img')
|
||||
|| query.img('.costume-bg', { attribute: 'data-img' })
|
||||
|| query.style('.model-profile, #profile, .carousel-item')?.['background-image']?.match(/url\((.*)\)/)?.[1];
|
||||
|| query.background('.model-profile, #profile, .carousel-item');
|
||||
|
||||
if (avatar) {
|
||||
profile.avatar = Array.from(new Set([
|
||||
|
||||
@@ -241,8 +241,8 @@ function generateActors(release) {
|
||||
: Math.floor(Math.random() * 3) + 2;
|
||||
|
||||
return Array.from({ length }, () => ({
|
||||
name: faker.name
|
||||
.findName()
|
||||
name: faker.person
|
||||
.fullName()
|
||||
.split(' ')
|
||||
.slice(0, Math.random() < 0.2 ? 1 : 2) // sometimes only use the first name
|
||||
.join(' '),
|
||||
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
|
||||
const knex = require('./knex');
|
||||
const { indexApi } = require('./manticore');
|
||||
const { HttpError } = require('./errors');
|
||||
const slugify = require('./utils/slugify');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
let lastActorsViewRefresh = 0;
|
||||
|
||||
function curateStash(stash) {
|
||||
if (!stash) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const curatedStash = {
|
||||
id: stash.id,
|
||||
name: stash.name,
|
||||
slug: stash.slug,
|
||||
primary: stash.primary,
|
||||
};
|
||||
|
||||
return curatedStash;
|
||||
}
|
||||
|
||||
function curateStashEntry(stash, user) {
|
||||
const curatedStashEntry = {
|
||||
user_id: user.id,
|
||||
name: stash.name,
|
||||
slug: slugify(stash.name),
|
||||
public: false,
|
||||
};
|
||||
|
||||
return curatedStashEntry;
|
||||
}
|
||||
|
||||
async function fetchStash(stashId, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const stash = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.first();
|
||||
|
||||
if (!stash) {
|
||||
throw new HttpError('You are not authorized to access this stash', 403);
|
||||
}
|
||||
|
||||
return curateStash(stash);
|
||||
}
|
||||
|
||||
async function fetchStashes(domain, itemId, sessionUser) {
|
||||
const stashes = await knex(`stashes_${domain}s`)
|
||||
.select('stashes.*')
|
||||
.where({
|
||||
[`${domain}_id`]: itemId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
||||
|
||||
return stashes.map((stash) => curateStash(stash));
|
||||
}
|
||||
|
||||
async function createStash(newStash, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
try {
|
||||
const stash = await knex('stashes')
|
||||
.insert(curateStashEntry(newStash, sessionUser))
|
||||
.returning('*');
|
||||
|
||||
return curateStash(stash);
|
||||
} catch (error) {
|
||||
if (error.routine === '_bt_check_unique') {
|
||||
throw new HttpError('Stash name should be unique', 409);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateStash(stashId, newStash, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const stash = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
})
|
||||
.update(newStash)
|
||||
.returning('*');
|
||||
|
||||
if (!stash) {
|
||||
throw new HttpError('You are not authorized to modify this stash', 403);
|
||||
}
|
||||
|
||||
return curateStash(stash);
|
||||
}
|
||||
|
||||
async function removeStash(stashId, sessionUser) {
|
||||
if (!sessionUser) {
|
||||
throw new HttpError('You are not authenthicated', 401);
|
||||
}
|
||||
|
||||
const removed = await knex('stashes')
|
||||
.where({
|
||||
id: stashId,
|
||||
user_id: sessionUser.id,
|
||||
primary: false,
|
||||
})
|
||||
.delete();
|
||||
|
||||
if (removed === 0) {
|
||||
throw new HttpError('Unable to remove this stash', 400);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshActorsView() {
|
||||
if (new Date() - lastActorsViewRefresh > config.stashes.viewRefreshCooldown * 60000) {
|
||||
// don't refresh actors view more than once an hour
|
||||
lastActorsViewRefresh = new Date();
|
||||
|
||||
logger.debug('Refreshing actors view');
|
||||
|
||||
return knex.schema.refreshMaterializedView('actors_meta');
|
||||
}
|
||||
|
||||
logger.silly('Skipping actors view refresh');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function stashActor(actorId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_actors')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
actor_id: actorId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'actors_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
actor_id: actorId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
refreshActorsView();
|
||||
|
||||
return fetchStashes('actor', actorId, sessionUser);
|
||||
}
|
||||
|
||||
async function stashScene(sceneId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_scenes')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
scene_id: sceneId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'scenes_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
// ...doc.replace.doc,
|
||||
scene_id: sceneId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('scene', sceneId, sessionUser);
|
||||
}
|
||||
|
||||
async function stashMovie(movieId, stashId, sessionUser) {
|
||||
const stash = await fetchStash(stashId, sessionUser);
|
||||
|
||||
const [stashed] = await knex('stashes_movies')
|
||||
.insert({
|
||||
stash_id: stash.id,
|
||||
movie_id: movieId,
|
||||
})
|
||||
.returning(['id', 'created_at']);
|
||||
|
||||
await indexApi.replace({
|
||||
index: 'movies_stashed',
|
||||
id: stashed.id,
|
||||
doc: {
|
||||
movie_id: movieId,
|
||||
user_id: sessionUser.id,
|
||||
stash_id: stashId,
|
||||
created_at: Math.round(stashed.created_at.getTime() / 1000),
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('movie', movieId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashActor(actorId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_actors AS deletable')
|
||||
.where('deletable.actor_id', actorId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_actors') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_actors.stash_id')
|
||||
.where('stashes_actors.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'actors_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { actor_id: actorId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
refreshActorsView();
|
||||
|
||||
return fetchStashes('actor', actorId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashScene(sceneId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_scenes AS deletable')
|
||||
.where('deletable.scene_id', sceneId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_scenes') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
||||
.where('stashes_scenes.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'scenes_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { scene_id: sceneId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('scene', sceneId, sessionUser);
|
||||
}
|
||||
|
||||
async function unstashMovie(movieId, stashId, sessionUser) {
|
||||
await knex
|
||||
.from('stashes_movies AS deletable')
|
||||
.where('deletable.movie_id', movieId)
|
||||
.where('deletable.stash_id', stashId)
|
||||
.whereExists(knex('stashes_movies') // verify user owns this stash, complimentary to row-level security
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_movies.stash_id')
|
||||
.where('stashes_movies.stash_id', knex.raw('deletable.stash_id'))
|
||||
.where('stashes.user_id', sessionUser.id))
|
||||
.delete();
|
||||
|
||||
await indexApi.callDelete({
|
||||
index: 'movies_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ equals: { movie_id: movieId } },
|
||||
{ equals: { stash_id: stashId } },
|
||||
{ equals: { user_id: sessionUser.id } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return fetchStashes('movie', movieId, sessionUser);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createStash,
|
||||
curateStash,
|
||||
removeStash,
|
||||
stashActor,
|
||||
stashScene,
|
||||
stashMovie,
|
||||
unstashScene,
|
||||
unstashActor,
|
||||
unstashMovie,
|
||||
updateStash,
|
||||
};
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const { decode } = require('html-entities');
|
||||
const { decode: decodeEntities } = require('html-entities');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const argv = require('./argv');
|
||||
@@ -28,6 +28,11 @@ const commonContext = {
|
||||
unprint,
|
||||
};
|
||||
|
||||
// newer html-entities throws on non-strings, pass e.g. numeric titles through like 2.4 did
|
||||
function decode(text) {
|
||||
return typeof text === 'string' ? decodeEntities(text) : text;
|
||||
}
|
||||
|
||||
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
||||
const slugBase = release.title
|
||||
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const fetch = require('node-fetch');
|
||||
const express = require('express');
|
||||
|
||||
async function init() {
|
||||
const res = await fetch('https://www.realitykings.com/scenes?site=45', {
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
'user-agent': 'HTTPie/2.6.0',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
accept: '*/*',
|
||||
connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
|
||||
console.log(res.status, res.headers);
|
||||
|
||||
const app = express();
|
||||
|
||||
app.get('/', (appReq, appRes) => {
|
||||
console.log(appReq.headers);
|
||||
appRes.status(204).send();
|
||||
});
|
||||
|
||||
app.listen(8000, () => {
|
||||
console.log('Listening on port 8000');
|
||||
|
||||
fetch('http://127.0.0.1:8000', {
|
||||
headers: {
|
||||
'user-agent': 'HTTPie/2.6.0',
|
||||
'accept-encoding': 'gzip, deflate, br',
|
||||
accept: '*/*',
|
||||
connection: 'keep-alive',
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,17 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const express = require('express');
|
||||
|
||||
const app = express();
|
||||
|
||||
app.get('/', (req, res) => {
|
||||
console.log(req.httpVersion);
|
||||
console.log(req.headers);
|
||||
console.log(req.body);
|
||||
|
||||
res.status(204).send();
|
||||
});
|
||||
|
||||
app.listen(5555, () => {
|
||||
console.log('Server started');
|
||||
});
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
// const util = require('util');
|
||||
const unprint = require('unprint');
|
||||
const args = require('yargs').argv;
|
||||
const args = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
|
||||
|
||||
const knex = require('../knex');
|
||||
const bulkInsert = require('../utils/bulk-insert');
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const knex = require('./knex');
|
||||
const { curateStash } = require('./stashes');
|
||||
|
||||
function curateUser(user) {
|
||||
if (!user) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ability = [...(user.role_abilities || []), ...(user.abilities || [])];
|
||||
|
||||
const curatedUser = {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
email: user.email,
|
||||
emailVerified: user.email_verified,
|
||||
identityVerified: user.identity_verified,
|
||||
ability,
|
||||
createdAt: user.created_at,
|
||||
stashes: user.stashes?.filter(Boolean).map((stash) => curateStash(stash)) || [],
|
||||
};
|
||||
|
||||
return curatedUser;
|
||||
}
|
||||
|
||||
async function fetchUser(userId, raw) {
|
||||
const user = await knex('users')
|
||||
.select(knex.raw('users.*, users_roles.abilities as role_abilities, COALESCE(json_agg(stashes ORDER BY stashes.created_at) FILTER (WHERE stashes.id IS NOT NULL), \'[]\') as stashes'))
|
||||
.modify((builder) => {
|
||||
if (typeof userId === 'number') {
|
||||
builder.where('users.id', userId);
|
||||
}
|
||||
|
||||
if (typeof userId === 'string') {
|
||||
builder
|
||||
.where('users.username', userId)
|
||||
.orWhere('users.email', userId);
|
||||
}
|
||||
})
|
||||
.leftJoin('users_roles', 'users_roles.role', 'users.role')
|
||||
.leftJoin('stashes', 'stashes.user_id', 'users.id')
|
||||
.groupBy('users.id', 'users_roles.role')
|
||||
.first();
|
||||
|
||||
if (raw) {
|
||||
return user;
|
||||
}
|
||||
|
||||
return curateUser(user);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
curateUser,
|
||||
fetchUser,
|
||||
};
|
||||
+1
-1
@@ -6,7 +6,7 @@ const fsPromises = require('fs').promises;
|
||||
const fs = require('fs');
|
||||
const { PassThrough } = require('stream');
|
||||
const crypto = require('crypto');
|
||||
const argv = require('yargs').argv;
|
||||
const argv = require('yargs')(require('yargs/helpers').hideBin(process.argv)).argv;
|
||||
|
||||
const file = 'https://speed.hetzner.de/100MB.bin';
|
||||
// const file = 'https://speed.hetzner.de/1GB.bin';
|
||||
|
||||
-153
@@ -1,153 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const arrayEqual = require('array-equal');
|
||||
|
||||
/* eslint-disable */
|
||||
function evaluateBranch(tree, modifiers) {
|
||||
const result = tree.map((expression) => {
|
||||
if (expression.type === 'group') {
|
||||
return evaluateBranch(expression.values, expression.modifiers);
|
||||
} if (expression.modifiers.length === 0) {
|
||||
return ''; // This is a trigger to stringify the previous values
|
||||
} if (arrayEqual(['plus'], expression.modifiers)) {
|
||||
return 0;
|
||||
} if (arrayEqual(['negate', 'negate'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['negate', 'plus'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['plus', 'plus', 'negate'], expression.modifiers)) {
|
||||
return true;
|
||||
} if (arrayEqual(['plus', 'negate', 'negate'], expression.modifiers)) {
|
||||
return 1;
|
||||
}
|
||||
throw new Error(`Found unrecognized modifier pattern: ${expression.modifiers}`);
|
||||
}).reduce((combined, value) => {
|
||||
if (value === '') {
|
||||
return combined.toString();
|
||||
}
|
||||
if (value === true) {
|
||||
value = 1;
|
||||
}
|
||||
|
||||
if (typeof combined === 'string') {
|
||||
return combined + value.toString();
|
||||
}
|
||||
return combined + value;
|
||||
}, 0);
|
||||
|
||||
if (modifiers == null) {
|
||||
return result;
|
||||
}
|
||||
if (arrayEqual(['plus'], modifiers)) {
|
||||
return parseInt(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function evaluate(tree) {
|
||||
return evaluateBranch(tree);
|
||||
}
|
||||
|
||||
function parse(string) {
|
||||
const length = string.length;
|
||||
let byte;
|
||||
|
||||
let stateFinishedItem = false;
|
||||
const modifierStack = [[]];
|
||||
const itemStack = [[]];
|
||||
let currentStack = itemStack[0];
|
||||
let currentModifiers = modifierStack[0];
|
||||
let stackLevel = 0;
|
||||
|
||||
for (let pos = 0; pos < length; pos++) {
|
||||
byte = string[pos];
|
||||
|
||||
switch (byte) {
|
||||
case '+':
|
||||
if (pos === 0 || stateFinishedItem === false) {
|
||||
// Modifier / number-cast
|
||||
currentModifiers.push('plus');
|
||||
stateFinishedItem = false;
|
||||
} else {
|
||||
// Addition, we don't need to do anything here
|
||||
}
|
||||
break;
|
||||
case '!':
|
||||
stateFinishedItem = false;
|
||||
currentModifiers.push('negate');
|
||||
break;
|
||||
case '(':
|
||||
stateFinishedItem = false;
|
||||
stackLevel++;
|
||||
itemStack[stackLevel] = currentStack = [];
|
||||
modifierStack[stackLevel] = currentModifiers = [];
|
||||
break;
|
||||
case ')':
|
||||
if (stackLevel === 0) {
|
||||
throw new Error('Encountered ) without matching (');
|
||||
}
|
||||
|
||||
stackLevel--;
|
||||
stateFinishedItem = true;
|
||||
|
||||
currentStack = itemStack[stackLevel];
|
||||
currentStack.push({
|
||||
type: 'group',
|
||||
values: itemStack[stackLevel + 1],
|
||||
modifiers: modifierStack[stackLevel],
|
||||
});
|
||||
|
||||
currentModifiers = modifierStack[stackLevel] = [];
|
||||
break;
|
||||
case '[':
|
||||
if (string[pos + 1] === ']') {
|
||||
// Reached the brackets; end of the modifier sequence
|
||||
currentStack.push({
|
||||
type: 'brackets',
|
||||
modifiers: currentModifiers,
|
||||
});
|
||||
|
||||
currentModifiers = [];
|
||||
pos += 1; // Skip over the closing bracket
|
||||
stateFinishedItem = true;
|
||||
} else {
|
||||
throw new Error(`Invalid byte found; expected ] but got ${string[pos + 1]}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return itemStack[0];
|
||||
}
|
||||
|
||||
function parseExpression(string) {
|
||||
return evaluate(parse(string));
|
||||
}
|
||||
|
||||
function findAll(regex, target) {
|
||||
const results = []; let
|
||||
match;
|
||||
|
||||
while (match = regex.exec(target)) {
|
||||
results.push(match);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function cf(testcase) {
|
||||
let [_, parent, child, initialExpression] = /var s,t,o,p,b,r,e,a,k,i,n,g,f,\s*([a-zA-Z]+)={"([a-zA-Z]+)":([^}]+)};/.exec(testcase);
|
||||
|
||||
const modifyingExpressions = findAll(new RegExp(`${parent}\.${child}\s*([*+-])=\s*([^;]+)`, 'g'), testcase).map(match => ({
|
||||
operation: match[1],
|
||||
expression: match[2],
|
||||
})).map(({ operation, expression }) => ({
|
||||
operation,
|
||||
expression: parseExpression(expression),
|
||||
}));
|
||||
|
||||
initialExpression = parseExpression(initialExpression);
|
||||
|
||||
return { parent, child, initialExpression, modifyingExpressions };
|
||||
};
|
||||
|
||||
module.exports = cf;
|
||||
@@ -82,7 +82,7 @@ function convertApi(rawInput, fromOrTo, to) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const input = decode(rawInput); // remove html entities, e.g. 5' 8" for 5' 8"
|
||||
const input = typeof rawInput === 'string' ? decode(rawInput) : rawInput; // remove html entities, e.g. 5' 8" for 5' 8"
|
||||
|
||||
try {
|
||||
if (typeof input === 'string' && to === undefined) {
|
||||
|
||||
+3
-269
@@ -2,16 +2,13 @@
|
||||
|
||||
const config = require('config');
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const undici = require('undici');
|
||||
const fs = require('fs').promises;
|
||||
// const util = require('util');
|
||||
// const stream = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
const Bottleneck = require('bottleneck');
|
||||
const { JSDOM, toughCookie } = require('jsdom');
|
||||
const puppeteer = require('puppeteer-extra');
|
||||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
const { JSDOM } = require('jsdom');
|
||||
|
||||
const windows = require('./http-windows');
|
||||
|
||||
@@ -21,24 +18,12 @@ const argv = require('../argv');
|
||||
|
||||
// const pipeline = util.promisify(stream.pipeline);
|
||||
|
||||
const limiters = {
|
||||
bypass: new Bottleneck({
|
||||
minTime: 1000,
|
||||
maxConcurrent: 1,
|
||||
timeout: 60000,
|
||||
}),
|
||||
};
|
||||
|
||||
const bypassSessions = new Map();
|
||||
|
||||
const browsers = new Map();
|
||||
const limiters = {};
|
||||
|
||||
Promise.config({
|
||||
cancellation: true,
|
||||
});
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
const defaultOptions = {
|
||||
timeout: argv.requestTimeout,
|
||||
encodeJSON: true,
|
||||
@@ -58,50 +43,6 @@ function useProxy(url) {
|
||||
return config.proxy.hostnames.includes(hostname);
|
||||
}
|
||||
|
||||
function useBrowserBypass(url, options) {
|
||||
if (!config.bypass.browser.enable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
if (options.bypassBrowser === 'shared') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (config.bypass.browser.hostnames.includes(hostname)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function useCloudflareBypass(url, options) {
|
||||
if (!config.bypass.cloudflare.enable) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { hostname } = new URL(url);
|
||||
|
||||
if (options.bypassCloudflare === 'shared') {
|
||||
return 'shared';
|
||||
}
|
||||
|
||||
if (options.bypassCloudflare === 'independent') {
|
||||
return hostname;
|
||||
}
|
||||
|
||||
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
|
||||
return 'shared';
|
||||
}
|
||||
|
||||
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
|
||||
return hostname;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const propMap = {
|
||||
media: {
|
||||
interval: 'mediaInterval',
|
||||
@@ -146,180 +87,6 @@ function getLimiter(options = {}, url) {
|
||||
return limiters[interval][concurrency];
|
||||
}
|
||||
|
||||
function extractJson(body, headers) {
|
||||
if (headers['content-type'].includes('application/json')) {
|
||||
const { document } = new JSDOM(body, { virtualConsole }).window;
|
||||
const dataString = document.querySelector('body > pre')?.textContent;
|
||||
|
||||
if (dataString) {
|
||||
const data = JSON.parse(dataString);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
async function getBrowserSession(identifier, options = {}) {
|
||||
return limiters.bypass.schedule(async () => {
|
||||
const browserIdentifier = options.useGlobalBrowser === false ? identifier : null;
|
||||
|
||||
if (!browsers.has(browserIdentifier)) {
|
||||
const newBrowser = await puppeteer.launch({
|
||||
headless: typeof options.headless === 'undefined' ? 'new' : options.headless,
|
||||
args: [
|
||||
...(options.useProxy && config.proxy.enabled ? [`--proxy-server=${config.proxy.host}:${config.proxy.port}`] : []),
|
||||
],
|
||||
// headless: false,
|
||||
});
|
||||
|
||||
browsers.set(browserIdentifier, newBrowser);
|
||||
|
||||
logger.info('Initialized puppeteer browser');
|
||||
}
|
||||
|
||||
const browser = browsers.get(browserIdentifier);
|
||||
const tab = await browser.newPage();
|
||||
|
||||
logger.verbose(`Opened puppeteer tab${identifier ? ` for ${identifier}` : ''}`);
|
||||
|
||||
return { browser, tab };
|
||||
});
|
||||
}
|
||||
|
||||
async function bypassBrowserRequest(url, options) {
|
||||
const { tab } = await getBrowserSession(new URL(url).hostname, options);
|
||||
|
||||
const res = await tab.goto(url);
|
||||
|
||||
if (options.bypass?.delay) {
|
||||
await Promise.delay(options.bypass.delay);
|
||||
}
|
||||
|
||||
if (typeof options.bypass?.evaluate === 'function') {
|
||||
await tab.evaluate(options.bypass.evaluate, options.bypass);
|
||||
}
|
||||
|
||||
const rawBody = await tab.content();
|
||||
|
||||
const headers = res.headers();
|
||||
const body = extractJson(rawBody, headers);
|
||||
|
||||
const statusCode = res.status();
|
||||
|
||||
if (!statusCode === 200) {
|
||||
throw new Error(`Puppeteer bypass failed for ${url} (${statusCode}): ${body?.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
body,
|
||||
statusCode,
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
async function getBypassSession(url, hostname) {
|
||||
if (bypassSessions.has(hostname)) {
|
||||
return bypassSessions.get(hostname);
|
||||
}
|
||||
|
||||
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.create',
|
||||
proxy: useProxy(url) ? {
|
||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
||||
} : null,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
});
|
||||
|
||||
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
|
||||
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
|
||||
}
|
||||
|
||||
bypassSessions.set(hostname, sessionRes.body.session);
|
||||
|
||||
return sessionRes.body.session;
|
||||
}
|
||||
|
||||
async function destroyBypassSession(sessionId) {
|
||||
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.destroy',
|
||||
session: sessionId,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
|
||||
bypassSessions.delete(sessionId);
|
||||
|
||||
logger.verbose(`Destroyed bypass session ${sessionId}`);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function destroyBypassSessions() {
|
||||
if (!config.bypass.cloudflare.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: 'sessions.list',
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
|
||||
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
|
||||
}
|
||||
|
||||
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
|
||||
}
|
||||
|
||||
async function destroyBrowserSessions() {
|
||||
await Promise.all(Array.from(browsers.values(), async (browser) => browser?.close()));
|
||||
}
|
||||
|
||||
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
|
||||
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
|
||||
|
||||
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
|
||||
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: `request.${method}`,
|
||||
url,
|
||||
session: sessionId,
|
||||
maxTimeout: options.timeout,
|
||||
proxy: useProxy(url) ? {
|
||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
||||
} : null,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
|
||||
if (/session closed/i.test(res.body?.message) && attempts < 3) {
|
||||
await destroyBypassSession(sessionId);
|
||||
|
||||
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
|
||||
}
|
||||
|
||||
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
|
||||
}
|
||||
|
||||
const resBody = extractJson(res.body.solution.response, res.body.solution.headers);
|
||||
|
||||
return {
|
||||
body: resBody,
|
||||
statusCode: res.body.solution.status,
|
||||
headers: res.body.solution.headers,
|
||||
};
|
||||
}
|
||||
|
||||
const defaultAgent = new undici.Agent({
|
||||
allowH2: true,
|
||||
connect: {
|
||||
@@ -333,19 +100,8 @@ const proxyAgent = new undici.ProxyAgent({
|
||||
|
||||
async function request(method = 'get', url, body, requestOptions = {}, limiter, redirects = 0) {
|
||||
const withProxy = useProxy(url);
|
||||
const withBrowserBypass = useBrowserBypass(url, requestOptions);
|
||||
const withCloudflareBypass = useCloudflareBypass(url, requestOptions);
|
||||
|
||||
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withBrowserBypass || withCloudflareBypass ? ' bypass' : ''}) ${url}`);
|
||||
|
||||
if (withBrowserBypass) {
|
||||
if (method !== 'get') throw new Error('Browser bypass only supports GET');
|
||||
return bypassBrowserRequest(url, requestOptions);
|
||||
}
|
||||
|
||||
if (withCloudflareBypass) {
|
||||
return bypassCloudflareRequest(url, method, body, withCloudflareBypass, requestOptions);
|
||||
}
|
||||
logger.debug(`${redirects > 0 ? 'REDIRECT ' : ''}${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
|
||||
|
||||
const headers = {
|
||||
...requestOptions.headers,
|
||||
@@ -479,33 +235,11 @@ async function head(url, options) {
|
||||
return scheduleRequest('head', url, null, options);
|
||||
}
|
||||
|
||||
function getSession(options) {
|
||||
return bhttp.session({ ...defaultOptions, ...options });
|
||||
}
|
||||
|
||||
function getCookieJar(store, options) {
|
||||
return new toughCookie.CookieJar(store, {
|
||||
looseMode: true,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
toughCookie,
|
||||
get,
|
||||
head,
|
||||
post,
|
||||
delete: del,
|
||||
put,
|
||||
patch,
|
||||
session: getSession,
|
||||
cookieJar: getCookieJar,
|
||||
browsers,
|
||||
getBrowserSession,
|
||||
getBypassSession,
|
||||
getSession,
|
||||
getCookieJar,
|
||||
destroyBypassSessions,
|
||||
destroyBrowserSessions,
|
||||
proxyAgent,
|
||||
};
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
const bhttp = require('bhttp');
|
||||
const sharp = require('sharp');
|
||||
const { argv } = require('yargs');
|
||||
const { argv } = require('yargs')(require('yargs/helpers').hideBin(process.argv));
|
||||
|
||||
const url = argv.url || 'http://localhost:5000/media/actors/tommy-pistol/1580341442712.jpeg';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user