forked from DebaucheryLibrarian/traxxx
Added CF resolver to http module. Using priority lookup in tags seed.
This commit is contained in:
DebaucheryLibrarian
parent
b7fbcdec94
commit
3db8b80164
|
|
@ -106,6 +106,7 @@ function popularEntities() {
|
|||
'pornpros',
|
||||
'private',
|
||||
'realitykings',
|
||||
'teamskeet',
|
||||
'twistys',
|
||||
'vixen',
|
||||
'xempire',
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@
|
|||
</div>
|
||||
|
||||
<div
|
||||
v-if="release.qualities"
|
||||
v-if="release.qualities?.length > 0"
|
||||
class="row"
|
||||
>
|
||||
<span class="row-label">Available qualities</span>
|
||||
|
|
|
|||
|
|
@ -292,6 +292,13 @@ module.exports = {
|
|||
'im0.imgcm.com',
|
||||
],
|
||||
},
|
||||
bypass: {
|
||||
cloudflare: {
|
||||
enable: true,
|
||||
auto: true, // try bypass when CF challenge is detected
|
||||
path: 'http://localhost:8191/v1',
|
||||
},
|
||||
},
|
||||
limits: {
|
||||
default: {
|
||||
interval: 50,
|
||||
|
|
|
|||
|
|
@ -99,7 +99,6 @@ const tags = [
|
|||
name: 'airtight',
|
||||
slug: 'airtight',
|
||||
description: 'Stuffing one cock in your ass, one in your pussy, and one in your mouth, filling up all of your penetrable holes and getting sealed airtight like a figurative balloon. In other words, simultaneously getting [double penetrated](/tag/dp), and giving a [blowjob](/tag/blowjob) or getting [facefucked](/tag/facefucking). Being airtight implies being [gangbanged](/tag/gangbang).', /* eslint-disable-line max-len */
|
||||
priority: 9,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
|
|
@ -109,7 +108,6 @@ const tags = [
|
|||
{
|
||||
name: 'anal creampie',
|
||||
slug: 'anal-creampie',
|
||||
priority: 7,
|
||||
description: 'Ejaculating into the asshole.',
|
||||
group: 'finish',
|
||||
},
|
||||
|
|
@ -117,7 +115,6 @@ const tags = [
|
|||
name: 'anal',
|
||||
slug: 'anal',
|
||||
description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.',
|
||||
priority: 9,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
|
|
@ -141,7 +138,6 @@ const tags = [
|
|||
{
|
||||
name: 'asian',
|
||||
slug: 'asian',
|
||||
priority: 5,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -152,14 +148,12 @@ const tags = [
|
|||
{
|
||||
name: 'ass to mouth',
|
||||
slug: 'atm',
|
||||
priority: 6,
|
||||
description: 'Sucking off a cock right after it fucked your ass. If it has been in someone else\'s ass, it may be called [ATOGM](/tag/atogm).',
|
||||
group: 'oral',
|
||||
},
|
||||
{
|
||||
name: 'ass to other girl\'s mouth',
|
||||
slug: 'atogm',
|
||||
priority: 6,
|
||||
description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".',
|
||||
group: 'oral',
|
||||
},
|
||||
|
|
@ -167,7 +161,6 @@ const tags = [
|
|||
name: 'ass eating',
|
||||
slug: 'ass-eating',
|
||||
group: 'oral',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'ass worship',
|
||||
|
|
@ -191,7 +184,6 @@ const tags = [
|
|||
{
|
||||
name: 'BDSM',
|
||||
slug: 'bdsm',
|
||||
priority: 8,
|
||||
},
|
||||
{
|
||||
name: 'BBC',
|
||||
|
|
@ -201,7 +193,6 @@ const tags = [
|
|||
{
|
||||
name: 'behind the scenes',
|
||||
slug: 'bts',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'big dick',
|
||||
|
|
@ -221,7 +212,6 @@ const tags = [
|
|||
{
|
||||
name: 'bisexual',
|
||||
slug: 'bisexual',
|
||||
priority: 10,
|
||||
},
|
||||
{
|
||||
name: 'black cock',
|
||||
|
|
@ -246,14 +236,12 @@ const tags = [
|
|||
{
|
||||
name: 'blowjob',
|
||||
slug: 'blowjob',
|
||||
priority: 5,
|
||||
description: 'Taking a dick in your mouth, sucking, licking and kissing it, often while giving a [handjob](/tag/handjob). You may slide it all the way [down your throat](/tag/deepthroat), or let them [fuck your face](/tag/facefucking).',
|
||||
group: 'oral',
|
||||
},
|
||||
{
|
||||
name: 'blowbang',
|
||||
slug: 'blowbang',
|
||||
priority: 9,
|
||||
description: 'Pleasuring a gang of three or more cocks by sucking and jerking off as many cocks as you can, often getting [facefucked](/tag/facefucking), groped and rubbed out, and followed by a [bukkake](/tag/bukkake). If you are also getting fucked, it is a [gangbang](/tag/gangbang).',
|
||||
group: 'group',
|
||||
},
|
||||
|
|
@ -273,7 +261,6 @@ const tags = [
|
|||
{
|
||||
name: 'bukkake',
|
||||
slug: 'bukkake',
|
||||
priority: 8,
|
||||
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
|
||||
group: 'finish',
|
||||
},
|
||||
|
|
@ -305,7 +292,6 @@ const tags = [
|
|||
{
|
||||
name: 'creampie',
|
||||
slug: 'creampie',
|
||||
priority: 8,
|
||||
description: 'Ejaculalating into her pussy, often shown visibly dripping out afterwards.',
|
||||
group: 'finish',
|
||||
},
|
||||
|
|
@ -355,64 +341,54 @@ const tags = [
|
|||
name: 'double anal',
|
||||
slug: 'dap',
|
||||
description: 'Two cocks filling up your ass at the same time. If there\'s a third cock in your pussy, it is [double anal triple penetration](/tag/da-tp).',
|
||||
priority: 8,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
name: 'double dildo',
|
||||
slug: 'double-dildo',
|
||||
description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'double dildo anal',
|
||||
slug: 'double-dildo-anal',
|
||||
description: 'Two people ass-fucking eachother with either end of a [double-sided dildo](/tag/double-dildo), "ass to ass".',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'double dildo DP',
|
||||
slug: 'double-dildo-dp',
|
||||
description: 'Using a [double-sided dildo](/tag/double-dildo) on your ass and pussy [at the same time](/tag/dp).',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'double dildo blowjob',
|
||||
slug: 'double-dildo-blowjob',
|
||||
description: 'Two people sucking and gagging on either end of a [double-sided dildo](/tag/double-dildo). They may deepthroat the dildo for a [double dildo kiss](/tag/double-dildo-kiss).',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'double dildo kiss',
|
||||
slug: 'double-dildo-kiss',
|
||||
description: 'Deepthroating a [double-sided dildo](/tag/double-dildo) during a [double dildo blowjob](/tag/double-dildo-blowjob), all the way until you can kiss eachother\'s lips.',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'triple anal',
|
||||
slug: 'tap',
|
||||
description: 'Getting fucked in the ass by not one, two, but *three* cocks at the same time.',
|
||||
priority: 8,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
name: 'triple vaginal',
|
||||
slug: 'tvp',
|
||||
description: 'Getting fucked in the pussy by *three* cocks at the same time.',
|
||||
priority: 8,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
name: 'deepthroat',
|
||||
slug: 'deepthroat',
|
||||
priority: 6,
|
||||
description: 'Shoving a cock down your throat during a [blowjob](/tag/blowjob) or [facefuck](/tag/facefucking), giving them a tight sensation while showing off your skills. Without practice, a cock hitting the back of your mouth will likely make you [gag](/tag/gagging).',
|
||||
group: 'oral',
|
||||
},
|
||||
{
|
||||
name: 'double penetration',
|
||||
slug: 'dp',
|
||||
priority: 9,
|
||||
description: 'Getting your [ass](/tag/anal) and pussy fucked at the same time. If you take another cock in your mouth, you are [airtight](/tag/airtight).',
|
||||
group: 'penetration',
|
||||
},
|
||||
|
|
@ -424,7 +400,6 @@ const tags = [
|
|||
name: 'double vaginal',
|
||||
slug: 'dvp',
|
||||
description: 'Getting fucked with two cocks in your pussy at the same time. If there\'s a third cock in your asshole, it is [double vaginal triple penetration](/tag/dv-tp).',
|
||||
priority: 8,
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
|
|
@ -450,7 +425,6 @@ const tags = [
|
|||
{
|
||||
name: 'black',
|
||||
slug: 'black',
|
||||
priority: 5,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -468,7 +442,6 @@ const tags = [
|
|||
{
|
||||
name: 'facefucking',
|
||||
slug: 'facefucking',
|
||||
priority: 7,
|
||||
description: 'A [blowjob](/tag/blowjob) where you give up control, and let them fuck your mouth and [throat](/tag/deepthroat) as if it\'s your pussy.',
|
||||
group: 'oral',
|
||||
},
|
||||
|
|
@ -485,7 +458,6 @@ const tags = [
|
|||
{
|
||||
name: 'family',
|
||||
slug: 'family',
|
||||
priority: 7,
|
||||
},
|
||||
{
|
||||
name: 'feet',
|
||||
|
|
@ -519,7 +491,6 @@ const tags = [
|
|||
{
|
||||
name: 'MFF threesome',
|
||||
slug: 'mff',
|
||||
priority: 9,
|
||||
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
|
||||
group: 'group',
|
||||
},
|
||||
|
|
@ -535,13 +506,11 @@ const tags = [
|
|||
name: 'gangbang',
|
||||
slug: 'gangbang',
|
||||
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
|
||||
priority: 9,
|
||||
group: 'group',
|
||||
},
|
||||
{
|
||||
name: 'older men',
|
||||
slug: 'older-men',
|
||||
priority: 7,
|
||||
group: 'age',
|
||||
},
|
||||
{
|
||||
|
|
@ -562,18 +531,15 @@ const tags = [
|
|||
name: 'trainbang',
|
||||
slug: 'trainbang',
|
||||
description: 'Getting fucked by a group of three or more guys taking turns in a [gangbang](/tag/gangbang), one cock after the other, and never more than one at a time.',
|
||||
priority: 7,
|
||||
group: 'group',
|
||||
},
|
||||
{
|
||||
name: 'gaping',
|
||||
slug: 'gaping',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'gay',
|
||||
slug: 'gay',
|
||||
priority: 10,
|
||||
},
|
||||
{
|
||||
name: 'gloryhole',
|
||||
|
|
@ -620,7 +586,6 @@ const tags = [
|
|||
{
|
||||
name: 'interracial',
|
||||
slug: 'interracial',
|
||||
priority: 7,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -634,7 +599,6 @@ const tags = [
|
|||
{
|
||||
name: 'Latina',
|
||||
slug: 'latina',
|
||||
priority: 5,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -644,7 +608,6 @@ const tags = [
|
|||
{
|
||||
name: 'lesbian',
|
||||
slug: 'lesbian',
|
||||
priority: 9,
|
||||
},
|
||||
{
|
||||
name: 'machine dildo',
|
||||
|
|
@ -671,7 +634,6 @@ const tags = [
|
|||
{
|
||||
name: 'male focus',
|
||||
slug: 'male-focus',
|
||||
priority: 7,
|
||||
description: 'Straight porn highlighting the male talent.',
|
||||
},
|
||||
{
|
||||
|
|
@ -681,13 +643,11 @@ const tags = [
|
|||
{
|
||||
name: 'MILF',
|
||||
slug: 'milf',
|
||||
priority: 7,
|
||||
group: 'age',
|
||||
},
|
||||
{
|
||||
name: 'MFM threesome',
|
||||
slug: 'mfm',
|
||||
priority: 9,
|
||||
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
|
||||
group: 'group',
|
||||
},
|
||||
|
|
@ -731,21 +691,18 @@ const tags = [
|
|||
{
|
||||
name: 'cum in mouth',
|
||||
slug: 'cum-in-mouth',
|
||||
priority: 7,
|
||||
description: 'A guy ejaculating in someone\'s mouth. If they keep their lips wrapped around his cock, it is an [oral creampie](/tag/oral-creampie). They may not be able to resist [swallowing](/tag/swallowing) the cum.',
|
||||
group: 'finish',
|
||||
},
|
||||
{
|
||||
name: 'oral creampie',
|
||||
slug: 'oral-creampie',
|
||||
priority: 7,
|
||||
description: 'A guy pumping his semen in someone\'s *closed* [mouth](/tag/cum-in-mouth), a variation of [cum in mouth](/tag/cum-in-mouth). His cock can be [deepthroated](/tag/deepthroat) for a [throatpie](/tag/throatpie)',
|
||||
group: 'finish',
|
||||
},
|
||||
{
|
||||
name: 'orgy',
|
||||
slug: 'orgy',
|
||||
priority: 9,
|
||||
description: 'A group of (at least four) people having sex with eachother. If only one person is getting fucked, it is probably a [gangbang](/tag/gangbang).',
|
||||
group: 'group',
|
||||
},
|
||||
|
|
@ -781,12 +738,10 @@ const tags = [
|
|||
{
|
||||
name: 'piss drinking',
|
||||
slug: 'piss-drinking',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'pissing',
|
||||
slug: 'pissing',
|
||||
priority: 8,
|
||||
},
|
||||
{
|
||||
name: 'POV',
|
||||
|
|
@ -800,7 +755,6 @@ const tags = [
|
|||
{
|
||||
name: 'pussy to mouth',
|
||||
slug: 'pussy-to-mouth',
|
||||
priority: 5,
|
||||
description: 'Sucking off a cock right fresh out of your pussy.',
|
||||
group: 'oral',
|
||||
},
|
||||
|
|
@ -829,7 +783,6 @@ const tags = [
|
|||
{
|
||||
name: 'rough',
|
||||
slug: 'rough',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'saliva',
|
||||
|
|
@ -939,7 +892,6 @@ const tags = [
|
|||
name: 'swallowing',
|
||||
slug: 'swallowing',
|
||||
group: 'finish',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'swinging',
|
||||
|
|
@ -959,13 +911,11 @@ const tags = [
|
|||
{
|
||||
name: 'threesome',
|
||||
slug: 'threesome',
|
||||
priority: 8,
|
||||
group: 'group',
|
||||
},
|
||||
{
|
||||
name: 'throatpie',
|
||||
slug: 'throatpie',
|
||||
priority: 6,
|
||||
description: 'An [oral creampie](/tag/oral-creampie) with his cock shoved all the way down the throat.',
|
||||
group: 'finish',
|
||||
},
|
||||
|
|
@ -982,43 +932,36 @@ const tags = [
|
|||
{
|
||||
name: 'toys',
|
||||
slug: 'toys',
|
||||
priority: 6,
|
||||
},
|
||||
{
|
||||
name: 'toy anal',
|
||||
slug: 'toy-anal',
|
||||
description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
|
||||
priority: 7,
|
||||
},
|
||||
{
|
||||
name: 'toy DP',
|
||||
slug: 'toy-dp',
|
||||
description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
|
||||
priority: 4,
|
||||
},
|
||||
{
|
||||
name: 'transsexual',
|
||||
slug: 'transsexual',
|
||||
priority: 10,
|
||||
},
|
||||
{
|
||||
name: 'DA triple penetration',
|
||||
slug: 'da-tp',
|
||||
priority: 7,
|
||||
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your ass](/tag/dap), and one in your pussy. Also see [double vaginal triple penetration](/tag/dv-tp).',
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
name: 'DV triple penetration',
|
||||
slug: 'dv-tp',
|
||||
priority: 7,
|
||||
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your pussy](/tag/dvp), and one in [your ass](/tag/anal). Also see [double anal triple penetration](/tag/da-tp).',
|
||||
group: 'penetration',
|
||||
},
|
||||
{
|
||||
name: 'triple penetration',
|
||||
slug: 'triple-penetration',
|
||||
priority: 7,
|
||||
description: 'Three cocks fucking her from behind at the same time. This can be either [double anal TP](/tag/da-tp), or [double vaginal TP](/tag/dv-tp).',
|
||||
},
|
||||
{
|
||||
|
|
@ -1054,7 +997,6 @@ const tags = [
|
|||
{
|
||||
name: 'Russian',
|
||||
slug: 'russian',
|
||||
priority: 4,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -1064,7 +1006,6 @@ const tags = [
|
|||
{
|
||||
name: 'white',
|
||||
slug: 'white',
|
||||
priority: 7,
|
||||
group: 'ethnicity',
|
||||
},
|
||||
{
|
||||
|
|
@ -1551,6 +1492,10 @@ const aliases = [
|
|||
for: 'tvp',
|
||||
secondary: true,
|
||||
},
|
||||
{
|
||||
name: 'double vag',
|
||||
for: 'dvp',
|
||||
},
|
||||
{
|
||||
name: 'double vaginal penetration',
|
||||
for: 'dvp',
|
||||
|
|
@ -1653,6 +1598,10 @@ const aliases = [
|
|||
name: 'french kissing',
|
||||
for: 'kissing',
|
||||
},
|
||||
{
|
||||
name: 'gang bang',
|
||||
for: 'gangbang',
|
||||
},
|
||||
{
|
||||
name: 'gape',
|
||||
for: 'gaping',
|
||||
|
|
@ -2049,6 +1998,27 @@ const aliases = [
|
|||
},
|
||||
];
|
||||
|
||||
const priorities = [ // higher index is higher priority
|
||||
['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
|
||||
['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
|
||||
['family'],
|
||||
['blowjob', 'deepthroat'],
|
||||
['asian', 'black', 'latina', 'white', 'interracial'],
|
||||
['bdsm', 'bts'],
|
||||
['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'],
|
||||
['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
|
||||
['lesbian', 'rough', 'milf', 'male-focus'],
|
||||
['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
|
||||
['anal', 'bukkake'],
|
||||
['dp', 'dap', 'tap', 'dvp', 'tvp', 'airtight'],
|
||||
['gangbang', 'blowbang', 'orgy'],
|
||||
['gay', 'bisexual'],
|
||||
].reduce((acc, slugs, index) => {
|
||||
slugs.forEach((slug) => { acc[slug] = index; });
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
exports.seed = (knex) => Promise.resolve()
|
||||
.then(async () => upsert('tags_groups', groups, 'slug', knex))
|
||||
.then(async () => {
|
||||
|
|
@ -2059,7 +2029,8 @@ exports.seed = (knex) => Promise.resolve()
|
|||
name: tag.name,
|
||||
slug: tag.slug || slugify(tag.name),
|
||||
description: tag.description,
|
||||
priority: tag.priority || 0,
|
||||
// priority: tag.priority || 0,
|
||||
priority: typeof priorities[tag.slug] === 'undefined' ? -1 : priorities[tag.slug],
|
||||
group_id: tag.group ? groupsMap[tag.group] : null,
|
||||
alias_for: null,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -119,7 +119,11 @@ function scrapeProfile(actor, entity) {
|
|||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
|
||||
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, {
|
||||
bypassCloudflare: true,
|
||||
});
|
||||
|
||||
console.log(res.status);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.body.hits.hits, channel);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,14 @@ const virtualConsole = require('./virtual-console')(__filename);
|
|||
const argv = require('../argv');
|
||||
|
||||
const pipeline = util.promisify(stream.pipeline);
|
||||
const limiters = {};
|
||||
|
||||
const limiters = {
|
||||
bypass: new Bottleneck({
|
||||
minTime: 1000,
|
||||
maxConcurrent: 1,
|
||||
timeout: 60000,
|
||||
}),
|
||||
};
|
||||
|
||||
Promise.config({
|
||||
cancellation: true,
|
||||
|
|
@ -84,6 +91,47 @@ function getLimiter(options = {}, url) {
|
|||
return limiters[interval][concurrency];
|
||||
}
|
||||
|
||||
function extractJson(solution) {
|
||||
if (solution.headers['content-type'].includes('application/json')) {
|
||||
const { document } = new JSDOM(solution.response, { virtualConsole }).window;
|
||||
const dataString = document.querySelector('body > pre')?.textContent;
|
||||
|
||||
if (dataString) {
|
||||
const data = JSON.parse(dataString);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
return solution.response;
|
||||
}
|
||||
|
||||
async function bypassCloudflareRequest(url, method, body, options) {
|
||||
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
|
||||
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
|
||||
cmd: `request.${method}`,
|
||||
url,
|
||||
maxTimeout: options.timeout,
|
||||
proxy: useProxy(url) ? {
|
||||
url: `${config.proxy.host}:${config.proxy.port}`,
|
||||
} : null,
|
||||
}, {
|
||||
encodeJSON: true,
|
||||
}));
|
||||
|
||||
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
|
||||
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
|
||||
}
|
||||
|
||||
const resBody = extractJson(res.body.solution);
|
||||
|
||||
return {
|
||||
body: resBody,
|
||||
statusCode: res.body.solution.status,
|
||||
headers: res.body.solution.headers,
|
||||
};
|
||||
}
|
||||
|
||||
async function request(method = 'get', url, body, requestOptions = {}, limiter) {
|
||||
const http = requestOptions.session || bhttp;
|
||||
|
||||
|
|
@ -93,12 +141,17 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
|
|||
};
|
||||
|
||||
const withProxy = useProxy(url);
|
||||
const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable;
|
||||
|
||||
if (withProxy) {
|
||||
options.agent = proxyAgent;
|
||||
}
|
||||
|
||||
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
|
||||
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
|
||||
|
||||
if (withCloudflareBypass) {
|
||||
return bypassCloudflareRequest(url, method, body, options);
|
||||
}
|
||||
|
||||
const res = await (body
|
||||
? http[method](url, body, options)
|
||||
|
|
|
|||
Loading…
Reference in New Issue