Compare commits

...

3 Commits

Author SHA1 Message Date
DebaucheryLibrarian e15f00e086 1.218.0 2022-09-27 20:09:53 +02:00
DebaucheryLibrarian 0fc37e46d2 Fixed media module trying to fetch invalid source URLs. Added Accidental Gangbang to Adult Time. 2022-09-27 20:09:46 +02:00
DebaucheryLibrarian 3db8b80164 Added CF resolver to http module. Using priority lookup in tags seed. 2022-08-15 23:51:51 +02:00
33 changed files with 4951 additions and 2509 deletions

View File

@ -106,6 +106,7 @@ function popularEntities() {
'pornpros',
'private',
'realitykings',
'teamskeet',
'twistys',
'vixen',
'xempire',

View File

@ -204,7 +204,7 @@
</div>
<div
v-if="release.qualities"
v-if="release.qualities?.length > 0"
class="row"
>
<span class="row-label">Available qualities</span>

View File

@ -431,6 +431,22 @@ const releasesFragment = `
}
}
}
or: [
{
entity: {
slug: {
notEqualTo: "analvids"
}
}
}
{
studio: {
slug: {
in: ["giorgiograndi"]
}
}
}
],
},
first: $limit,
offset: $offset,

View File

@ -292,6 +292,18 @@ module.exports = {
'im0.imgcm.com',
],
},
bypass: {
cloudflare: {
enable: true,
auto: true, // try bypass when CF challenge is detected
path: 'http://localhost:8191/v1',
sharedHostnames: [ // these can run in the same browser session
'store2.psmcdn.net', // Team Skeet API
'www.kink.com',
],
independentHostnames: [], // these must run in their own browser session
},
},
limits: {
default: {
interval: 50,

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.217.3",
"version": "1.218.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.217.3",
"version": "1.218.0",
"license": "ISC",
"dependencies": {
"@casl/ability": "^5.2.2",

View File

@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.217.3",
"version": "1.218.0",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.0 KiB

After

Width:  |  Height:  |  Size: 7.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.3 KiB

After

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.1 KiB

After

Width:  |  Height:  |  Size: 5.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

View File

@ -99,7 +99,6 @@ const tags = [
name: 'airtight',
slug: 'airtight',
description: 'Stuffing one cock in your ass, one in your pussy, and one in your mouth, filling up all of your penetrable holes and getting sealed airtight like a figurative balloon. In other words, simultaneously getting [double penetrated](/tag/dp), and giving a [blowjob](/tag/blowjob) or getting [facefucked](/tag/facefucking). Being airtight implies being [gangbanged](/tag/gangbang).', /* eslint-disable-line max-len */
priority: 9,
group: 'penetration',
},
{
@ -109,7 +108,6 @@ const tags = [
{
name: 'anal creampie',
slug: 'anal-creampie',
priority: 7,
description: 'Ejaculating into the asshole.',
group: 'finish',
},
@ -117,7 +115,6 @@ const tags = [
name: 'anal',
slug: 'anal',
description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.',
priority: 9,
group: 'penetration',
},
{
@ -141,7 +138,6 @@ const tags = [
{
name: 'asian',
slug: 'asian',
priority: 5,
group: 'ethnicity',
},
{
@ -152,14 +148,12 @@ const tags = [
{
name: 'ass to mouth',
slug: 'atm',
priority: 6,
description: 'Sucking off a cock right after it fucked your ass. If it has been in someone else\'s ass, it may be called [ATOGM](/tag/atogm).',
group: 'oral',
},
{
name: 'ass to other girl\'s mouth',
slug: 'atogm',
priority: 6,
description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".',
group: 'oral',
},
@ -167,7 +161,6 @@ const tags = [
name: 'ass eating',
slug: 'ass-eating',
group: 'oral',
priority: 6,
},
{
name: 'ass worship',
@ -191,7 +184,6 @@ const tags = [
{
name: 'BDSM',
slug: 'bdsm',
priority: 8,
},
{
name: 'BBC',
@ -201,7 +193,6 @@ const tags = [
{
name: 'behind the scenes',
slug: 'bts',
priority: 6,
},
{
name: 'big dick',
@ -221,7 +212,6 @@ const tags = [
{
name: 'bisexual',
slug: 'bisexual',
priority: 10,
},
{
name: 'black cock',
@ -246,14 +236,12 @@ const tags = [
{
name: 'blowjob',
slug: 'blowjob',
priority: 5,
description: 'Taking a dick in your mouth, sucking, licking and kissing it, often while giving a [handjob](/tag/handjob). You may slide it all the way [down your throat](/tag/deepthroat), or let them [fuck your face](/tag/facefucking).',
group: 'oral',
},
{
name: 'blowbang',
slug: 'blowbang',
priority: 9,
description: 'Pleasuring a gang of three or more cocks by sucking and jerking off as many cocks as you can, often getting [facefucked](/tag/facefucking), groped and rubbed out, and followed by a [bukkake](/tag/bukkake). If you are also getting fucked, it is a [gangbang](/tag/gangbang).',
group: 'group',
},
@ -273,7 +261,6 @@ const tags = [
{
name: 'bukkake',
slug: 'bukkake',
priority: 8,
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
group: 'finish',
},
@ -305,7 +292,6 @@ const tags = [
{
name: 'creampie',
slug: 'creampie',
priority: 8,
description: 'Ejaculalating into her pussy, often shown visibly dripping out afterwards.',
group: 'finish',
},
@ -355,64 +341,54 @@ const tags = [
name: 'double anal',
slug: 'dap',
description: 'Two cocks filling up your ass at the same time. If there\'s a third cock in your pussy, it is [double anal triple penetration](/tag/da-tp).',
priority: 8,
group: 'penetration',
},
{
name: 'double dildo',
slug: 'double-dildo',
description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).',
priority: 4,
},
{
name: 'double dildo anal',
slug: 'double-dildo-anal',
description: 'Two people ass-fucking eachother with either end of a [double-sided dildo](/tag/double-dildo), "ass to ass".',
priority: 4,
},
{
name: 'double dildo DP',
slug: 'double-dildo-dp',
description: 'Using a [double-sided dildo](/tag/double-dildo) on your ass and pussy [at the same time](/tag/dp).',
priority: 4,
},
{
name: 'double dildo blowjob',
slug: 'double-dildo-blowjob',
description: 'Two people sucking and gagging on either end of a [double-sided dildo](/tag/double-dildo). They may deepthroat the dildo for a [double dildo kiss](/tag/double-dildo-kiss).',
priority: 4,
},
{
name: 'double dildo kiss',
slug: 'double-dildo-kiss',
description: 'Deepthroating a [double-sided dildo](/tag/double-dildo) during a [double dildo blowjob](/tag/double-dildo-blowjob), all the way until you can kiss eachother\'s lips.',
priority: 4,
},
{
name: 'triple anal',
slug: 'tap',
description: 'Getting fucked in the ass by not one, two, but *three* cocks at the same time.',
priority: 8,
group: 'penetration',
},
{
name: 'triple vaginal',
slug: 'tvp',
description: 'Getting fucked in the pussy by *three* cocks at the same time.',
priority: 8,
group: 'penetration',
},
{
name: 'deepthroat',
slug: 'deepthroat',
priority: 6,
description: 'Shoving a cock down your throat during a [blowjob](/tag/blowjob) or [facefuck](/tag/facefucking), giving them a tight sensation while showing off your skills. Without practice, a cock hitting the back of your mouth will likely make you [gag](/tag/gagging).',
group: 'oral',
},
{
name: 'double penetration',
slug: 'dp',
priority: 9,
description: 'Getting your [ass](/tag/anal) and pussy fucked at the same time. If you take another cock in your mouth, you are [airtight](/tag/airtight).',
group: 'penetration',
},
@ -424,7 +400,6 @@ const tags = [
name: 'double vaginal',
slug: 'dvp',
description: 'Getting fucked with two cocks in your pussy at the same time. If there\'s a third cock in your asshole, it is [double vaginal triple penetration](/tag/dv-tp).',
priority: 8,
group: 'penetration',
},
{
@ -450,7 +425,6 @@ const tags = [
{
name: 'black',
slug: 'black',
priority: 5,
group: 'ethnicity',
},
{
@ -468,7 +442,6 @@ const tags = [
{
name: 'facefucking',
slug: 'facefucking',
priority: 7,
description: 'A [blowjob](/tag/blowjob) where you give up control, and let them fuck your mouth and [throat](/tag/deepthroat) as if it\'s your pussy.',
group: 'oral',
},
@ -485,7 +458,6 @@ const tags = [
{
name: 'family',
slug: 'family',
priority: 7,
},
{
name: 'feet',
@ -519,7 +491,6 @@ const tags = [
{
name: 'MFF threesome',
slug: 'mff',
priority: 9,
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
group: 'group',
},
@ -535,13 +506,11 @@ const tags = [
name: 'gangbang',
slug: 'gangbang',
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
priority: 9,
group: 'group',
},
{
name: 'older men',
slug: 'older-men',
priority: 7,
group: 'age',
},
{
@ -562,18 +531,15 @@ const tags = [
name: 'trainbang',
slug: 'trainbang',
description: 'Getting fucked by a group of three or more guys taking turns in a [gangbang](/tag/gangbang), one cock after the other, and never more than one at a time.',
priority: 7,
group: 'group',
},
{
name: 'gaping',
slug: 'gaping',
priority: 6,
},
{
name: 'gay',
slug: 'gay',
priority: 10,
},
{
name: 'gloryhole',
@ -620,7 +586,6 @@ const tags = [
{
name: 'interracial',
slug: 'interracial',
priority: 7,
group: 'ethnicity',
},
{
@ -634,7 +599,6 @@ const tags = [
{
name: 'Latina',
slug: 'latina',
priority: 5,
group: 'ethnicity',
},
{
@ -644,7 +608,6 @@ const tags = [
{
name: 'lesbian',
slug: 'lesbian',
priority: 9,
},
{
name: 'machine dildo',
@ -671,7 +634,6 @@ const tags = [
{
name: 'male focus',
slug: 'male-focus',
priority: 7,
description: 'Straight porn highlighting the male talent.',
},
{
@ -681,13 +643,11 @@ const tags = [
{
name: 'MILF',
slug: 'milf',
priority: 7,
group: 'age',
},
{
name: 'MFM threesome',
slug: 'mfm',
priority: 9,
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
group: 'group',
},
@ -731,21 +691,18 @@ const tags = [
{
name: 'cum in mouth',
slug: 'cum-in-mouth',
priority: 7,
description: 'A guy ejaculating in someone\'s mouth. If they keep their lips wrapped around his cock, it is an [oral creampie](/tag/oral-creampie). They may not be able to resist [swallowing](/tag/swallowing) the cum.',
group: 'finish',
},
{
name: 'oral creampie',
slug: 'oral-creampie',
priority: 7,
description: 'A guy pumping his semen in someone\'s *closed* [mouth](/tag/cum-in-mouth), a variation of [cum in mouth](/tag/cum-in-mouth). His cock can be [deepthroated](/tag/deepthroat) for a [throatpie](/tag/throatpie)',
group: 'finish',
},
{
name: 'orgy',
slug: 'orgy',
priority: 9,
description: 'A group of (at least four) people having sex with eachother. If only one person is getting fucked, it is probably a [gangbang](/tag/gangbang).',
group: 'group',
},
@ -781,12 +738,10 @@ const tags = [
{
name: 'piss drinking',
slug: 'piss-drinking',
priority: 6,
},
{
name: 'pissing',
slug: 'pissing',
priority: 8,
},
{
name: 'POV',
@ -800,7 +755,6 @@ const tags = [
{
name: 'pussy to mouth',
slug: 'pussy-to-mouth',
priority: 5,
description: 'Sucking off a cock right fresh out of your pussy.',
group: 'oral',
},
@ -829,7 +783,6 @@ const tags = [
{
name: 'rough',
slug: 'rough',
priority: 6,
},
{
name: 'saliva',
@ -939,7 +892,6 @@ const tags = [
name: 'swallowing',
slug: 'swallowing',
group: 'finish',
priority: 6,
},
{
name: 'swinging',
@ -959,13 +911,11 @@ const tags = [
{
name: 'threesome',
slug: 'threesome',
priority: 8,
group: 'group',
},
{
name: 'throatpie',
slug: 'throatpie',
priority: 6,
description: 'An [oral creampie](/tag/oral-creampie) with his cock shoved all the way down the throat.',
group: 'finish',
},
@ -982,43 +932,36 @@ const tags = [
{
name: 'toys',
slug: 'toys',
priority: 6,
},
{
name: 'toy anal',
slug: 'toy-anal',
description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
priority: 7,
},
{
name: 'toy DP',
slug: 'toy-dp',
description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
priority: 4,
},
{
name: 'transsexual',
slug: 'transsexual',
priority: 10,
},
{
name: 'DA triple penetration',
slug: 'da-tp',
priority: 7,
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your ass](/tag/dap), and one in your pussy. Also see [double vaginal triple penetration](/tag/dv-tp).',
group: 'penetration',
},
{
name: 'DV triple penetration',
slug: 'dv-tp',
priority: 7,
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your pussy](/tag/dvp), and one in [your ass](/tag/anal). Also see [double anal triple penetration](/tag/da-tp).',
group: 'penetration',
},
{
name: 'triple penetration',
slug: 'triple-penetration',
priority: 7,
description: 'Three cocks fucking her from behind at the same time. This can be either [double anal TP](/tag/da-tp), or [double vaginal TP](/tag/dv-tp).',
},
{
@ -1054,7 +997,6 @@ const tags = [
{
name: 'Russian',
slug: 'russian',
priority: 4,
group: 'ethnicity',
},
{
@ -1064,7 +1006,6 @@ const tags = [
{
name: 'white',
slug: 'white',
priority: 7,
group: 'ethnicity',
},
{
@ -1551,6 +1492,10 @@ const aliases = [
for: 'tvp',
secondary: true,
},
{
name: 'double vag',
for: 'dvp',
},
{
name: 'double vaginal penetration',
for: 'dvp',
@ -1653,6 +1598,10 @@ const aliases = [
name: 'french kissing',
for: 'kissing',
},
{
name: 'gang bang',
for: 'gangbang',
},
{
name: 'gape',
for: 'gaping',
@ -2049,6 +1998,27 @@ const aliases = [
},
];
const priorities = [ // higher index is higher priority
['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
['family'],
['blowjob', 'deepthroat'],
['asian', 'black', 'latina', 'white', 'interracial'],
['bdsm', 'bts'],
['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'],
['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
['lesbian', 'rough', 'milf', 'male-focus'],
['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
['anal', 'bukkake'],
['dp', 'dap', 'tap', 'dvp', 'tvp', 'airtight'],
['gangbang', 'blowbang', 'orgy'],
['gay', 'bisexual'],
].reduce((acc, slugs, index) => {
slugs.forEach((slug) => { acc[slug] = index; });
return acc;
}, {});
exports.seed = (knex) => Promise.resolve()
.then(async () => upsert('tags_groups', groups, 'slug', knex))
.then(async () => {
@ -2059,7 +2029,8 @@ exports.seed = (knex) => Promise.resolve()
name: tag.name,
slug: tag.slug || slugify(tag.name),
description: tag.description,
priority: tag.priority || 0,
// priority: tag.priority || 0,
priority: typeof priorities[tag.slug] === 'undefined' ? -1 : priorities[tag.slug],
group_id: tag.group ? groupsMap[tag.group] : null,
alias_for: null,
}));

View File

@ -363,6 +363,19 @@ const sites = [
},
},
// ADULT TIME
{
name: 'Accidental Gangbang',
slug: 'accidentalgangbang',
url: 'https://accidentalgangbang.com',
parent: 'adulttime',
tags: ['gangbang'],
parameters: {
referer: 'https://freetour.adulttime.com/en/join',
deep: 'https://21sextury.com/en/video',
scene: false,
includePhotos: false,
},
},
{
name: 'ASMR Fantasy',
slug: 'asmrfantasy',

File diff suppressed because it is too large Load Diff

View File

@ -199,6 +199,8 @@ async function init() {
logger.error(error);
}
await http.destroyBypassSessions();
knex.destroy();
done = true;
}

View File

@ -96,15 +96,39 @@ function itemsByKey(items, key) {
return items.reduce((acc, item) => ({ ...acc, [item[key]]: item }), {});
}
function isValidUrl(url) {
try {
const urlObject = new URL(url);
return !!urlObject;
} catch (error) {
return false;
}
}
function toBaseSource(rawSource) {
if (rawSource && (rawSource.src || (rawSource.extract && rawSource.url) || rawSource.stream)) {
const baseSource = {};
if (rawSource.src) baseSource.src = rawSource.src;
if (rawSource.src) {
if (!isValidUrl(rawSource.src)) {
return null;
}
baseSource.src = rawSource.src;
}
if (rawSource.quality) baseSource.quality = rawSource.quality;
if (rawSource.type) baseSource.type = rawSource.type;
if (rawSource.url) baseSource.url = rawSource.url;
if (rawSource.url) {
if (!isValidUrl(rawSource.url)) {
return null;
}
baseSource.url = rawSource.url;
}
if (rawSource.extract) baseSource.extract = rawSource.extract;
if (rawSource.expectType) baseSource.expectType = rawSource.expectType;
@ -135,6 +159,10 @@ function toBaseSource(rawSource) {
}
if (typeof rawSource === 'string') {
if (!isValidUrl(rawSource)) {
return null;
}
return {
src: rawSource,
};

View File

@ -150,7 +150,7 @@ async function getFullPhotos(entryId, site) {
},
});
if (res.ok) {
if (res.ok && typeof res.body === 'object') { // gives 200 OK even when redirected to signup page
return Object.values(res.body);
}
@ -442,7 +442,7 @@ async function scrapeReleaseApi(data, site, options) {
];
}
if (data.photoset_id && options.includePhotos) {
if (data.photoset_id && options.includePhotos && options.parameters?.includePhotos !== false) {
release.photos = await getPhotosApi(data.photoset_id, site, options.parameters);
}

View File

@ -132,13 +132,9 @@ async function scrapeProfile({ query }, actorUrl, include) {
async function fetchLatest(site, page = 1) {
// const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot', {
const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
Host: 'www.kink.com',
'User-Agent': 'HTTPie/2.6.0',
'Accept-Encoding': 'gzip, deflate, br',
Accept: '*/*',
Connection: 'keep-alive',
// const res = await qu.getAll(`https://www.kink.com/channel/bound-gang-bangs/latest/page/${page}`, '.shoot-list .shoot', {
const res = await qu.getAll(`https://www.kink.com/search?type=shoots&channelIds=${site.slug}&sort=published&page=${page}`, '.shoot-list .shoot', {
ct: 2,
}, {
includeDefaultHeaders: false,
followRedirects: false,

File diff suppressed because it is too large Load Diff

View File

@ -17,7 +17,16 @@ const virtualConsole = require('./virtual-console')(__filename);
const argv = require('../argv');
const pipeline = util.promisify(stream.pipeline);
const limiters = {};
const limiters = {
bypass: new Bottleneck({
minTime: 1000,
maxConcurrent: 1,
timeout: 60000,
}),
};
const bypassSessions = new Map();
Promise.config({
cancellation: true,
@ -49,6 +58,32 @@ function useProxy(url) {
return config.proxy.hostnames.includes(hostname);
}
function useCloudflareBypass(url, options) {
if (!config.bypass.cloudflare.enable) {
return null;
}
const { hostname } = new URL(url);
if (options.bypassCloudflare === 'shared') {
return 'shared';
}
if (options.bypassCloudflare === 'independent') {
return hostname;
}
if (config.bypass.cloudflare.sharedHostnames.includes(hostname)) {
return 'shared';
}
if (config.bypass.cloudflare.independentHostnames.includes(hostname)) {
return hostname;
}
return null;
}
function getLimiterValue(prop, options, hostname) {
if (argv[prop] !== undefined) {
return argv[prop];
@ -84,6 +119,114 @@ function getLimiter(options = {}, url) {
return limiters[interval][concurrency];
}
function extractJson(solution) {
if (solution.headers['content-type'].includes('application/json')) {
const { document } = new JSDOM(solution.response, { virtualConsole }).window;
const dataString = document.querySelector('body > pre')?.textContent;
if (dataString) {
const data = JSON.parse(dataString);
return data;
}
}
return solution.response;
}
async function getBypassSession(url, hostname) {
if (bypassSessions.has(hostname)) {
return bypassSessions.get(hostname);
}
const sessionRes = await bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.create',
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
});
if (sessionRes.statusCode !== 200 || sessionRes.body.status !== 'ok') {
throw new Error(`Could not acquire CloudFlare bypass session for ${url} (${sessionRes.statusCode}): ${sessionRes.body?.message}`);
}
bypassSessions.set(hostname, sessionRes.body.session);
return sessionRes.body.session;
}
async function destroyBypassSession(sessionId) {
const sessionDestroyRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.destroy',
session: sessionId,
}, {
encodeJSON: true,
}));
if (sessionDestroyRes.statusCode === 200 && sessionDestroyRes.body.status === 'ok') {
bypassSessions.delete(sessionId);
logger.verbose(`Destroyed bypass session ${sessionId}`);
return true;
}
logger.warn(`Failed to destroy bypass session ${sessionId} (${sessionDestroyRes.statusCode}): ${sessionDestroyRes.body?.message}`);
return false;
}
async function destroyBypassSessions() {
const sessionListRes = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: 'sessions.list',
}, {
encodeJSON: true,
}));
if (sessionListRes.statusCode !== 200 && sessionListRes.body.status !== 'ok') {
logger.warn(`Failed to remove bypass sessions (${sessionListRes.statusCode}): ${sessionListRes.body?.message}`);
}
await Promise.map(sessionListRes.body.sessions, async (sessionId) => destroyBypassSession(sessionId), { concurrency: 5 });
}
async function bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts = 0) {
const sessionId = await limiters.bypass.schedule(async () => getBypassSession(url, cloudflareBypass));
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: `request.${method}`,
url,
session: sessionId,
maxTimeout: options.timeout,
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
}));
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
if (/session closed/i.test(res.body?.message) && attempts < 3) {
await destroyBypassSession(sessionId);
return bypassCloudflareRequest(url, method, body, cloudflareBypass, options, attempts + 1);
}
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
}
const resBody = extractJson(res.body.solution);
return {
body: resBody,
statusCode: res.body.solution.status,
headers: res.body.solution.headers,
};
}
async function request(method = 'get', url, body, requestOptions = {}, limiter) {
const http = requestOptions.session || bhttp;
@ -93,12 +236,17 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
};
const withProxy = useProxy(url);
const withCloudflareBypass = useCloudflareBypass(url, options);
if (withProxy) {
options.agent = proxyAgent;
}
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
if (withCloudflareBypass) {
return bypassCloudflareRequest(url, method, body, withCloudflareBypass, options);
}
const res = await (body
? http[method](url, body, options)
@ -239,4 +387,5 @@ module.exports = {
cookieJar: getCookieJar,
getSession,
getCookieJar,
destroyBypassSessions,
};