From 3db8b801645f6759f12d5e74bf2416edd3d309a2 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Mon, 15 Aug 2022 23:51:51 +0200 Subject: [PATCH] Added CF resolver to http module. Using priority lookup in tags seed. --- assets/components/networks/networks.vue | 1 + assets/components/releases/release.vue | 2 +- config/default.js | 7 ++ seeds/00_tags.js | 91 +++++++++---------------- src/scrapers/teamskeet.js | 6 +- src/utils/http.js | 57 +++++++++++++++- 6 files changed, 100 insertions(+), 64 deletions(-) diff --git a/assets/components/networks/networks.vue b/assets/components/networks/networks.vue index 2083c360e..eb2823715 100644 --- a/assets/components/networks/networks.vue +++ b/assets/components/networks/networks.vue @@ -106,6 +106,7 @@ function popularEntities() { 'pornpros', 'private', 'realitykings', + 'teamskeet', 'twistys', 'vixen', 'xempire', diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index b077d800a..c379559bd 100644 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -204,7 +204,7 @@
Available qualities diff --git a/config/default.js b/config/default.js index 26d8b1131..950495f10 100644 --- a/config/default.js +++ b/config/default.js @@ -292,6 +292,13 @@ module.exports = { 'im0.imgcm.com', ], }, + bypass: { + cloudflare: { + enable: true, + auto: true, // try bypass when CF challenge is detected + path: 'http://localhost:8191/v1', + }, + }, limits: { default: { interval: 50, diff --git a/seeds/00_tags.js b/seeds/00_tags.js index e0529f956..f0a044af0 100644 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -99,7 +99,6 @@ const tags = [ name: 'airtight', slug: 'airtight', description: 'Stuffing one cock in your ass, one in your pussy, and one in your mouth, filling up all of your penetrable holes and getting sealed airtight like a figurative balloon. In other words, simultaneously getting [double penetrated](/tag/dp), and giving a [blowjob](/tag/blowjob) or getting [facefucked](/tag/facefucking). Being airtight implies being [gangbanged](/tag/gangbang).', /* eslint-disable-line max-len */ - priority: 9, group: 'penetration', }, { @@ -109,7 +108,6 @@ const tags = [ { name: 'anal creampie', slug: 'anal-creampie', - priority: 7, description: 'Ejaculating into the asshole.', group: 'finish', }, @@ -117,7 +115,6 @@ const tags = [ name: 'anal', slug: 'anal', description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.', - priority: 9, group: 'penetration', }, { @@ -141,7 +138,6 @@ const tags = [ { name: 'asian', slug: 'asian', - priority: 5, group: 'ethnicity', }, { @@ -152,14 +148,12 @@ const tags = [ { name: 'ass to mouth', slug: 'atm', - priority: 6, description: 'Sucking off a cock right after it fucked your ass. If it has been in someone else\'s ass, it may be called [ATOGM](/tag/atogm).', group: 'oral', }, { name: 'ass to other girl\'s mouth', slug: 'atogm', - priority: 6, description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".', group: 'oral', }, @@ -167,7 +161,6 @@ const tags = [ name: 'ass eating', slug: 'ass-eating', group: 'oral', - priority: 6, }, { name: 'ass worship', @@ -191,7 +184,6 @@ const tags = [ { name: 'BDSM', slug: 'bdsm', - priority: 8, }, { name: 'BBC', @@ -201,7 +193,6 @@ const tags = [ { name: 'behind the scenes', slug: 'bts', - priority: 6, }, { name: 'big dick', @@ -221,7 +212,6 @@ const tags = [ { name: 'bisexual', slug: 'bisexual', - priority: 10, }, { name: 'black cock', @@ -246,14 +236,12 @@ const tags = [ { name: 'blowjob', slug: 'blowjob', - priority: 5, description: 'Taking a dick in your mouth, sucking, licking and kissing it, often while giving a [handjob](/tag/handjob). You may slide it all the way [down your throat](/tag/deepthroat), or let them [fuck your face](/tag/facefucking).', group: 'oral', }, { name: 'blowbang', slug: 'blowbang', - priority: 9, description: 'Pleasuring a gang of three or more cocks by sucking and jerking off as many cocks as you can, often getting [facefucked](/tag/facefucking), groped and rubbed out, and followed by a [bukkake](/tag/bukkake). If you are also getting fucked, it is a [gangbang](/tag/gangbang).', group: 'group', }, @@ -273,7 +261,6 @@ const tags = [ { name: 'bukkake', slug: 'bukkake', - priority: 8, description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).', group: 'finish', }, @@ -305,7 +292,6 @@ const tags = [ { name: 'creampie', slug: 'creampie', - priority: 8, description: 'Ejaculalating into her pussy, often shown visibly dripping out afterwards.', group: 'finish', }, @@ -355,64 +341,54 @@ const tags = [ name: 'double anal', slug: 'dap', description: 'Two cocks filling up your ass at the same time. If there\'s a third cock in your pussy, it is [double anal triple penetration](/tag/da-tp).', - priority: 8, group: 'penetration', }, { name: 'double dildo', slug: 'double-dildo', description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).', - priority: 4, }, { name: 'double dildo anal', slug: 'double-dildo-anal', description: 'Two people ass-fucking eachother with either end of a [double-sided dildo](/tag/double-dildo), "ass to ass".', - priority: 4, }, { name: 'double dildo DP', slug: 'double-dildo-dp', description: 'Using a [double-sided dildo](/tag/double-dildo) on your ass and pussy [at the same time](/tag/dp).', - priority: 4, }, { name: 'double dildo blowjob', slug: 'double-dildo-blowjob', description: 'Two people sucking and gagging on either end of a [double-sided dildo](/tag/double-dildo). They may deepthroat the dildo for a [double dildo kiss](/tag/double-dildo-kiss).', - priority: 4, }, { name: 'double dildo kiss', slug: 'double-dildo-kiss', description: 'Deepthroating a [double-sided dildo](/tag/double-dildo) during a [double dildo blowjob](/tag/double-dildo-blowjob), all the way until you can kiss eachother\'s lips.', - priority: 4, }, { name: 'triple anal', slug: 'tap', description: 'Getting fucked in the ass by not one, two, but *three* cocks at the same time.', - priority: 8, group: 'penetration', }, { name: 'triple vaginal', slug: 'tvp', description: 'Getting fucked in the pussy by *three* cocks at the same time.', - priority: 8, group: 'penetration', }, { name: 'deepthroat', slug: 'deepthroat', - priority: 6, description: 'Shoving a cock down your throat during a [blowjob](/tag/blowjob) or [facefuck](/tag/facefucking), giving them a tight sensation while showing off your skills. Without practice, a cock hitting the back of your mouth will likely make you [gag](/tag/gagging).', group: 'oral', }, { name: 'double penetration', slug: 'dp', - priority: 9, description: 'Getting your [ass](/tag/anal) and pussy fucked at the same time. If you take another cock in your mouth, you are [airtight](/tag/airtight).', group: 'penetration', }, @@ -424,7 +400,6 @@ const tags = [ name: 'double vaginal', slug: 'dvp', description: 'Getting fucked with two cocks in your pussy at the same time. If there\'s a third cock in your asshole, it is [double vaginal triple penetration](/tag/dv-tp).', - priority: 8, group: 'penetration', }, { @@ -450,7 +425,6 @@ const tags = [ { name: 'black', slug: 'black', - priority: 5, group: 'ethnicity', }, { @@ -468,7 +442,6 @@ const tags = [ { name: 'facefucking', slug: 'facefucking', - priority: 7, description: 'A [blowjob](/tag/blowjob) where you give up control, and let them fuck your mouth and [throat](/tag/deepthroat) as if it\'s your pussy.', group: 'oral', }, @@ -485,7 +458,6 @@ const tags = [ { name: 'family', slug: 'family', - priority: 7, }, { name: 'feet', @@ -519,7 +491,6 @@ const tags = [ { name: 'MFF threesome', slug: 'mff', - priority: 9, description: 'A threesome with two women and one guy, in which the women have sex with eachother.', group: 'group', }, @@ -535,13 +506,11 @@ const tags = [ name: 'gangbang', slug: 'gangbang', description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */ - priority: 9, group: 'group', }, { name: 'older men', slug: 'older-men', - priority: 7, group: 'age', }, { @@ -562,18 +531,15 @@ const tags = [ name: 'trainbang', slug: 'trainbang', description: 'Getting fucked by a group of three or more guys taking turns in a [gangbang](/tag/gangbang), one cock after the other, and never more than one at a time.', - priority: 7, group: 'group', }, { name: 'gaping', slug: 'gaping', - priority: 6, }, { name: 'gay', slug: 'gay', - priority: 10, }, { name: 'gloryhole', @@ -620,7 +586,6 @@ const tags = [ { name: 'interracial', slug: 'interracial', - priority: 7, group: 'ethnicity', }, { @@ -634,7 +599,6 @@ const tags = [ { name: 'Latina', slug: 'latina', - priority: 5, group: 'ethnicity', }, { @@ -644,7 +608,6 @@ const tags = [ { name: 'lesbian', slug: 'lesbian', - priority: 9, }, { name: 'machine dildo', @@ -671,7 +634,6 @@ const tags = [ { name: 'male focus', slug: 'male-focus', - priority: 7, description: 'Straight porn highlighting the male talent.', }, { @@ -681,13 +643,11 @@ const tags = [ { name: 'MILF', slug: 'milf', - priority: 7, group: 'age', }, { name: 'MFM threesome', slug: 'mfm', - priority: 9, description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.', group: 'group', }, @@ -731,21 +691,18 @@ const tags = [ { name: 'cum in mouth', slug: 'cum-in-mouth', - priority: 7, description: 'A guy ejaculating in someone\'s mouth. If they keep their lips wrapped around his cock, it is an [oral creampie](/tag/oral-creampie). They may not be able to resist [swallowing](/tag/swallowing) the cum.', group: 'finish', }, { name: 'oral creampie', slug: 'oral-creampie', - priority: 7, description: 'A guy pumping his semen in someone\'s *closed* [mouth](/tag/cum-in-mouth), a variation of [cum in mouth](/tag/cum-in-mouth). His cock can be [deepthroated](/tag/deepthroat) for a [throatpie](/tag/throatpie)', group: 'finish', }, { name: 'orgy', slug: 'orgy', - priority: 9, description: 'A group of (at least four) people having sex with eachother. If only one person is getting fucked, it is probably a [gangbang](/tag/gangbang).', group: 'group', }, @@ -781,12 +738,10 @@ const tags = [ { name: 'piss drinking', slug: 'piss-drinking', - priority: 6, }, { name: 'pissing', slug: 'pissing', - priority: 8, }, { name: 'POV', @@ -800,7 +755,6 @@ const tags = [ { name: 'pussy to mouth', slug: 'pussy-to-mouth', - priority: 5, description: 'Sucking off a cock right fresh out of your pussy.', group: 'oral', }, @@ -829,7 +783,6 @@ const tags = [ { name: 'rough', slug: 'rough', - priority: 6, }, { name: 'saliva', @@ -939,7 +892,6 @@ const tags = [ name: 'swallowing', slug: 'swallowing', group: 'finish', - priority: 6, }, { name: 'swinging', @@ -959,13 +911,11 @@ const tags = [ { name: 'threesome', slug: 'threesome', - priority: 8, group: 'group', }, { name: 'throatpie', slug: 'throatpie', - priority: 6, description: 'An [oral creampie](/tag/oral-creampie) with his cock shoved all the way down the throat.', group: 'finish', }, @@ -982,43 +932,36 @@ const tags = [ { name: 'toys', slug: 'toys', - priority: 6, }, { name: 'toy anal', slug: 'toy-anal', description: 'Stuffing a toy, such as a dildo or buttplug, into the ass', - priority: 7, }, { name: 'toy DP', slug: 'toy-dp', description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).', - priority: 4, }, { name: 'transsexual', slug: 'transsexual', - priority: 10, }, { name: 'DA triple penetration', slug: 'da-tp', - priority: 7, description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your ass](/tag/dap), and one in your pussy. Also see [double vaginal triple penetration](/tag/dv-tp).', group: 'penetration', }, { name: 'DV triple penetration', slug: 'dv-tp', - priority: 7, description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your pussy](/tag/dvp), and one in [your ass](/tag/anal). Also see [double anal triple penetration](/tag/da-tp).', group: 'penetration', }, { name: 'triple penetration', slug: 'triple-penetration', - priority: 7, description: 'Three cocks fucking her from behind at the same time. This can be either [double anal TP](/tag/da-tp), or [double vaginal TP](/tag/dv-tp).', }, { @@ -1054,7 +997,6 @@ const tags = [ { name: 'Russian', slug: 'russian', - priority: 4, group: 'ethnicity', }, { @@ -1064,7 +1006,6 @@ const tags = [ { name: 'white', slug: 'white', - priority: 7, group: 'ethnicity', }, { @@ -1551,6 +1492,10 @@ const aliases = [ for: 'tvp', secondary: true, }, + { + name: 'double vag', + for: 'dvp', + }, { name: 'double vaginal penetration', for: 'dvp', @@ -1653,6 +1598,10 @@ const aliases = [ name: 'french kissing', for: 'kissing', }, + { + name: 'gang bang', + for: 'gangbang', + }, { name: 'gape', for: 'gaping', @@ -2049,6 +1998,27 @@ const aliases = [ }, ]; +const priorities = [ // higher index is higher priority + ['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'], + ['toys', 'toy-anal', 'toy-dp', 'piss-drinking'], + ['family'], + ['blowjob', 'deepthroat'], + ['asian', 'black', 'latina', 'white', 'interracial'], + ['bdsm', 'bts'], + ['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'], + ['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'], + ['lesbian', 'rough', 'milf', 'male-focus'], + ['threesome', 'mfm', 'mff', 'trainbang', 'pissing'], + ['anal', 'bukkake'], + ['dp', 'dap', 'tap', 'dvp', 'tvp', 'airtight'], + ['gangbang', 'blowbang', 'orgy'], + ['gay', 'bisexual'], +].reduce((acc, slugs, index) => { + slugs.forEach((slug) => { acc[slug] = index; }); + + return acc; +}, {}); + exports.seed = (knex) => Promise.resolve() .then(async () => upsert('tags_groups', groups, 'slug', knex)) .then(async () => { @@ -2059,7 +2029,8 @@ exports.seed = (knex) => Promise.resolve() name: tag.name, slug: tag.slug || slugify(tag.name), description: tag.description, - priority: tag.priority || 0, + // priority: tag.priority || 0, + priority: typeof priorities[tag.slug] === 'undefined' ? -1 : priorities[tag.slug], group_id: tag.group ? groupsMap[tag.group] : null, alias_for: null, })); diff --git a/src/scrapers/teamskeet.js b/src/scrapers/teamskeet.js index c1026a0dc..1043525f5 100644 --- a/src/scrapers/teamskeet.js +++ b/src/scrapers/teamskeet.js @@ -119,7 +119,11 @@ function scrapeProfile(actor, entity) { } async function fetchLatest(channel, page = 1, { parameters }) { - const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); + const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, { + bypassCloudflare: true, + }); + + console.log(res.status); if (res.ok) { return scrapeAll(res.body.hits.hits, channel); diff --git a/src/utils/http.js b/src/utils/http.js index 98c1b304a..e11c8d93e 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -17,7 +17,14 @@ const virtualConsole = require('./virtual-console')(__filename); const argv = require('../argv'); const pipeline = util.promisify(stream.pipeline); -const limiters = {}; + +const limiters = { + bypass: new Bottleneck({ + minTime: 1000, + maxConcurrent: 1, + timeout: 60000, + }), +}; Promise.config({ cancellation: true, @@ -84,6 +91,47 @@ function getLimiter(options = {}, url) { return limiters[interval][concurrency]; } +function extractJson(solution) { + if (solution.headers['content-type'].includes('application/json')) { + const { document } = new JSDOM(solution.response, { virtualConsole }).window; + const dataString = document.querySelector('body > pre')?.textContent; + + if (dataString) { + const data = JSON.parse(dataString); + + return data; + } + } + + return solution.response; +} + +async function bypassCloudflareRequest(url, method, body, options) { + // the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL + const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, { + cmd: `request.${method}`, + url, + maxTimeout: options.timeout, + proxy: useProxy(url) ? { + url: `${config.proxy.host}:${config.proxy.port}`, + } : null, + }, { + encodeJSON: true, + })); + + if (!res.statusCode === 200 || res.body?.status !== 'ok') { + throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`); + } + + const resBody = extractJson(res.body.solution); + + return { + body: resBody, + statusCode: res.body.solution.status, + headers: res.body.solution.headers, + }; +} + async function request(method = 'get', url, body, requestOptions = {}, limiter) { const http = requestOptions.session || bhttp; @@ -93,12 +141,17 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter) }; const withProxy = useProxy(url); + const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable; if (withProxy) { options.agent = proxyAgent; } - logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`); + logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`); + + if (withCloudflareBypass) { + return bypassCloudflareRequest(url, method, body, options); + } const res = await (body ? http[method](url, body, options)