DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added CF resolver to http module. Using priority lookup in tags seed.

This commit is contained in:
DebaucheryLibrarian 2022-08-15 23:51:51 +02:00
parent b7fbcdec94
commit 3db8b80164
6 changed files with 100 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
assets/components/networks/networks.vue
View File

@ -106,6 +106,7 @@ function popularEntities() {
'pornpros',
'private',
'realitykings',
'teamskeet',
'twistys',
'vixen',
'xempire',

2
assets/components/releases/release.vue
View File

@ -204,7 +204,7 @@
</div>
<div
v-if="release.qualities"
v-if="release.qualities?.length > 0"
class="row"
>
<span class="row-label">Available qualities</span>

7
config/default.js
View File

@ -292,6 +292,13 @@ module.exports = {
'im0.imgcm.com',
],
},
bypass: {
cloudflare: {
enable: true,
auto: true, // try bypass when CF challenge is detected
path: 'http://localhost:8191/v1',
},
},
limits: {
default: {
interval: 50,

91
seeds/00_tags.js
View File

@ -99,7 +99,6 @@ const tags = [
name: 'airtight',
slug: 'airtight',
description: 'Stuffing one cock in your ass, one in your pussy, and one in your mouth, filling up all of your penetrable holes and getting sealed airtight like a figurative balloon. In other words, simultaneously getting [double penetrated](/tag/dp), and giving a [blowjob](/tag/blowjob) or getting [facefucked](/tag/facefucking). Being airtight implies being [gangbanged](/tag/gangbang).', /* eslint-disable-line max-len */
priority: 9,
group: 'penetration',
},
{
@ -109,7 +108,6 @@ const tags = [
{
name: 'anal creampie',
slug: 'anal-creampie',
priority: 7,
description: 'Ejaculating into the asshole.',
group: 'finish',
},
@ -117,7 +115,6 @@ const tags = [
name: 'anal',
slug: 'anal',
description: 'Getting your asshole fucked. Generally considered naughtier, you may or may not find it a pleasurable alternative to vaginal sex.',
priority: 9,
group: 'penetration',
},
{
@ -141,7 +138,6 @@ const tags = [
{
name: 'asian',
slug: 'asian',
priority: 5,
group: 'ethnicity',
},
{
@ -152,14 +148,12 @@ const tags = [
{
name: 'ass to mouth',
slug: 'atm',
priority: 6,
description: 'Sucking off a cock right after it fucked your ass. If it has been in someone else\'s ass, it may be called [ATOGM](/tag/atogm).',
group: 'oral',
},
{
name: 'ass to other girl\'s mouth',
slug: 'atogm',
priority: 6,
description: '[Ass to mouth](/tag/atm) with a cock that has been in someone else\'s ass. ATOGM may also be the gay variation "ass to other guy\'s mouth".',
group: 'oral',
},
@ -167,7 +161,6 @@ const tags = [
name: 'ass eating',
slug: 'ass-eating',
group: 'oral',
priority: 6,
},
{
name: 'ass worship',
@ -191,7 +184,6 @@ const tags = [
{
name: 'BDSM',
slug: 'bdsm',
priority: 8,
},
{
name: 'BBC',
@ -201,7 +193,6 @@ const tags = [
{
name: 'behind the scenes',
slug: 'bts',
priority: 6,
},
{
name: 'big dick',
@ -221,7 +212,6 @@ const tags = [
{
name: 'bisexual',
slug: 'bisexual',
priority: 10,
},
{
name: 'black cock',
@ -246,14 +236,12 @@ const tags = [
{
name: 'blowjob',
slug: 'blowjob',
priority: 5,
description: 'Taking a dick in your mouth, sucking, licking and kissing it, often while giving a [handjob](/tag/handjob). You may slide it all the way [down your throat](/tag/deepthroat), or let them [fuck your face](/tag/facefucking).',
group: 'oral',
},
{
name: 'blowbang',
slug: 'blowbang',
priority: 9,
description: 'Pleasuring a gang of three or more cocks by sucking and jerking off as many cocks as you can, often getting [facefucked](/tag/facefucking), groped and rubbed out, and followed by a [bukkake](/tag/bukkake). If you are also getting fucked, it is a [gangbang](/tag/gangbang).',
group: 'group',
},
@ -273,7 +261,6 @@ const tags = [
{
name: 'bukkake',
slug: 'bukkake',
priority: 8,
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
group: 'finish',
},
@ -305,7 +292,6 @@ const tags = [
{
name: 'creampie',
slug: 'creampie',
priority: 8,
description: 'Ejaculalating into her pussy, often shown visibly dripping out afterwards.',
group: 'finish',
},
@ -355,64 +341,54 @@ const tags = [
name: 'double anal',
slug: 'dap',
description: 'Two cocks filling up your ass at the same time. If there\'s a third cock in your pussy, it is [double anal triple penetration](/tag/da-tp).',
priority: 8,
group: 'penetration',
},
{
name: 'double dildo',
slug: 'double-dildo',
description: 'Two girls fucking eachother using either end of a double-sided dildo. They can suck it for a [double dildo blowjob](/tag/double-dildo-blowjob), deepthroat it for a [double dildo kiss](/tag/double-dildo-kiss), or put it up their ass for [double dildo anal](/tag/double-dildo-anal).',
priority: 4,
},
{
name: 'double dildo anal',
slug: 'double-dildo-anal',
description: 'Two people ass-fucking eachother with either end of a [double-sided dildo](/tag/double-dildo), "ass to ass".',
priority: 4,
},
{
name: 'double dildo DP',
slug: 'double-dildo-dp',
description: 'Using a [double-sided dildo](/tag/double-dildo) on your ass and pussy [at the same time](/tag/dp).',
priority: 4,
},
{
name: 'double dildo blowjob',
slug: 'double-dildo-blowjob',
description: 'Two people sucking and gagging on either end of a [double-sided dildo](/tag/double-dildo). They may deepthroat the dildo for a [double dildo kiss](/tag/double-dildo-kiss).',
priority: 4,
},
{
name: 'double dildo kiss',
slug: 'double-dildo-kiss',
description: 'Deepthroating a [double-sided dildo](/tag/double-dildo) during a [double dildo blowjob](/tag/double-dildo-blowjob), all the way until you can kiss eachother\'s lips.',
priority: 4,
},
{
name: 'triple anal',
slug: 'tap',
description: 'Getting fucked in the ass by not one, two, but *three* cocks at the same time.',
priority: 8,
group: 'penetration',
},
{
name: 'triple vaginal',
slug: 'tvp',
description: 'Getting fucked in the pussy by *three* cocks at the same time.',
priority: 8,
group: 'penetration',
},
{
name: 'deepthroat',
slug: 'deepthroat',
priority: 6,
description: 'Shoving a cock down your throat during a [blowjob](/tag/blowjob) or [facefuck](/tag/facefucking), giving them a tight sensation while showing off your skills. Without practice, a cock hitting the back of your mouth will likely make you [gag](/tag/gagging).',
group: 'oral',
},
{
name: 'double penetration',
slug: 'dp',
priority: 9,
description: 'Getting your [ass](/tag/anal) and pussy fucked at the same time. If you take another cock in your mouth, you are [airtight](/tag/airtight).',
group: 'penetration',
},
@ -424,7 +400,6 @@ const tags = [
name: 'double vaginal',
slug: 'dvp',
description: 'Getting fucked with two cocks in your pussy at the same time. If there\'s a third cock in your asshole, it is [double vaginal triple penetration](/tag/dv-tp).',
priority: 8,
group: 'penetration',
},
{
@ -450,7 +425,6 @@ const tags = [
{
name: 'black',
slug: 'black',
priority: 5,
group: 'ethnicity',
},
{
@ -468,7 +442,6 @@ const tags = [
{
name: 'facefucking',
slug: 'facefucking',
priority: 7,
description: 'A [blowjob](/tag/blowjob) where you give up control, and let them fuck your mouth and [throat](/tag/deepthroat) as if it\'s your pussy.',
group: 'oral',
},
@ -485,7 +458,6 @@ const tags = [
{
name: 'family',
slug: 'family',
priority: 7,
},
{
name: 'feet',
@ -519,7 +491,6 @@ const tags = [
{
name: 'MFF threesome',
slug: 'mff',
priority: 9,
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
group: 'group',
},
@ -535,13 +506,11 @@ const tags = [
name: 'gangbang',
slug: 'gangbang',
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
priority: 9,
group: 'group',
},
{
name: 'older men',
slug: 'older-men',
priority: 7,
group: 'age',
},
{
@ -562,18 +531,15 @@ const tags = [
name: 'trainbang',
slug: 'trainbang',
description: 'Getting fucked by a group of three or more guys taking turns in a [gangbang](/tag/gangbang), one cock after the other, and never more than one at a time.',
priority: 7,
group: 'group',
},
{
name: 'gaping',
slug: 'gaping',
priority: 6,
},
{
name: 'gay',
slug: 'gay',
priority: 10,
},
{
name: 'gloryhole',
@ -620,7 +586,6 @@ const tags = [
{
name: 'interracial',
slug: 'interracial',
priority: 7,
group: 'ethnicity',
},
{
@ -634,7 +599,6 @@ const tags = [
{
name: 'Latina',
slug: 'latina',
priority: 5,
group: 'ethnicity',
},
{
@ -644,7 +608,6 @@ const tags = [
{
name: 'lesbian',
slug: 'lesbian',
priority: 9,
},
{
name: 'machine dildo',
@ -671,7 +634,6 @@ const tags = [
{
name: 'male focus',
slug: 'male-focus',
priority: 7,
description: 'Straight porn highlighting the male talent.',
},
{
@ -681,13 +643,11 @@ const tags = [
{
name: 'MILF',
slug: 'milf',
priority: 7,
group: 'age',
},
{
name: 'MFM threesome',
slug: 'mfm',
priority: 9,
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
group: 'group',
},
@ -731,21 +691,18 @@ const tags = [
{
name: 'cum in mouth',
slug: 'cum-in-mouth',
priority: 7,
description: 'A guy ejaculating in someone\'s mouth. If they keep their lips wrapped around his cock, it is an [oral creampie](/tag/oral-creampie). They may not be able to resist [swallowing](/tag/swallowing) the cum.',
group: 'finish',
},
{
name: 'oral creampie',
slug: 'oral-creampie',
priority: 7,
description: 'A guy pumping his semen in someone\'s *closed* [mouth](/tag/cum-in-mouth), a variation of [cum in mouth](/tag/cum-in-mouth). His cock can be [deepthroated](/tag/deepthroat) for a [throatpie](/tag/throatpie)',
group: 'finish',
},
{
name: 'orgy',
slug: 'orgy',
priority: 9,
description: 'A group of (at least four) people having sex with eachother. If only one person is getting fucked, it is probably a [gangbang](/tag/gangbang).',
group: 'group',
},
@ -781,12 +738,10 @@ const tags = [
{
name: 'piss drinking',
slug: 'piss-drinking',
priority: 6,
},
{
name: 'pissing',
slug: 'pissing',
priority: 8,
},
{
name: 'POV',
@ -800,7 +755,6 @@ const tags = [
{
name: 'pussy to mouth',
slug: 'pussy-to-mouth',
priority: 5,
description: 'Sucking off a cock right fresh out of your pussy.',
group: 'oral',
},
@ -829,7 +783,6 @@ const tags = [
{
name: 'rough',
slug: 'rough',
priority: 6,
},
{
name: 'saliva',
@ -939,7 +892,6 @@ const tags = [
name: 'swallowing',
slug: 'swallowing',
group: 'finish',
priority: 6,
},
{
name: 'swinging',
@ -959,13 +911,11 @@ const tags = [
{
name: 'threesome',
slug: 'threesome',
priority: 8,
group: 'group',
},
{
name: 'throatpie',
slug: 'throatpie',
priority: 6,
description: 'An [oral creampie](/tag/oral-creampie) with his cock shoved all the way down the throat.',
group: 'finish',
},
@ -982,43 +932,36 @@ const tags = [
{
name: 'toys',
slug: 'toys',
priority: 6,
},
{
name: 'toy anal',
slug: 'toy-anal',
description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
priority: 7,
},
{
name: 'toy DP',
slug: 'toy-dp',
description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
priority: 4,
},
{
name: 'transsexual',
slug: 'transsexual',
priority: 10,
},
{
name: 'DA triple penetration',
slug: 'da-tp',
priority: 7,
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your ass](/tag/dap), and one in your pussy. Also see [double vaginal triple penetration](/tag/dv-tp).',
group: 'penetration',
},
{
name: 'DV triple penetration',
slug: 'dv-tp',
priority: 7,
description: '[Triple penetration](/tag/triple-penetration) with [two cocks in your pussy](/tag/dvp), and one in [your ass](/tag/anal). Also see [double anal triple penetration](/tag/da-tp).',
group: 'penetration',
},
{
name: 'triple penetration',
slug: 'triple-penetration',
priority: 7,
description: 'Three cocks fucking her from behind at the same time. This can be either [double anal TP](/tag/da-tp), or [double vaginal TP](/tag/dv-tp).',
},
{
@ -1054,7 +997,6 @@ const tags = [
{
name: 'Russian',
slug: 'russian',
priority: 4,
group: 'ethnicity',
},
{
@ -1064,7 +1006,6 @@ const tags = [
{
name: 'white',
slug: 'white',
priority: 7,
group: 'ethnicity',
},
{
@ -1551,6 +1492,10 @@ const aliases = [
for: 'tvp',
secondary: true,
},
{
name: 'double vag',
for: 'dvp',
},
{
name: 'double vaginal penetration',
for: 'dvp',
@ -1653,6 +1598,10 @@ const aliases = [
name: 'french kissing',
for: 'kissing',
},
{
name: 'gang bang',
for: 'gangbang',
},
{
name: 'gape',
for: 'gaping',
@ -2049,6 +1998,27 @@ const aliases = [
},
];
const priorities = [ // higher index is higher priority
['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
['family'],
['blowjob', 'deepthroat'],
['asian', 'black', 'latina', 'white', 'interracial'],
['bdsm', 'bts'],
['facefucking', 'gaping', 'atm', 'atogm', 'pussy-to-mouth', 'ass-eating'],
['facial', 'swallowing', 'creampie', 'anal-creampie', 'oral-creampie', 'cum-in-mouth', 'throatpie'],
['lesbian', 'rough', 'milf', 'male-focus'],
['threesome', 'mfm', 'mff', 'trainbang', 'pissing'],
['anal', 'bukkake'],
['dp', 'dap', 'tap', 'dvp', 'tvp', 'airtight'],
['gangbang', 'blowbang', 'orgy'],
['gay', 'bisexual'],
].reduce((acc, slugs, index) => {
slugs.forEach((slug) => { acc[slug] = index; });
return acc;
}, {});
exports.seed = (knex) => Promise.resolve()
.then(async () => upsert('tags_groups', groups, 'slug', knex))
.then(async () => {
@ -2059,7 +2029,8 @@ exports.seed = (knex) => Promise.resolve()
name: tag.name,
slug: tag.slug || slugify(tag.name),
description: tag.description,
priority: tag.priority || 0,
// priority: tag.priority || 0,
priority: typeof priorities[tag.slug] === 'undefined' ? -1 : priorities[tag.slug],
group_id: tag.group ? groupsMap[tag.group] : null,
alias_for: null,
}));

6
src/scrapers/teamskeet.js
View File

@ -119,7 +119,11 @@ function scrapeProfile(actor, entity) {
}
async function fetchLatest(channel, page = 1, { parameters }) {
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`);
const res = await http.get(`${parameters.videos}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`, {
bypassCloudflare: true,
});
console.log(res.status);
if (res.ok) {
return scrapeAll(res.body.hits.hits, channel);

57
src/utils/http.js
View File

@ -17,7 +17,14 @@ const virtualConsole = require('./virtual-console')(__filename);
const argv = require('../argv');
const pipeline = util.promisify(stream.pipeline);
const limiters = {};
const limiters = {
bypass: new Bottleneck({
minTime: 1000,
maxConcurrent: 1,
timeout: 60000,
}),
};
Promise.config({
cancellation: true,
@ -84,6 +91,47 @@ function getLimiter(options = {}, url) {
return limiters[interval][concurrency];
}
function extractJson(solution) {
if (solution.headers['content-type'].includes('application/json')) {
const { document } = new JSDOM(solution.response, { virtualConsole }).window;
const dataString = document.querySelector('body > pre')?.textContent;
if (dataString) {
const data = JSON.parse(dataString);
return data;
}
}
return solution.response;
}
async function bypassCloudflareRequest(url, method, body, options) {
// the bypass proxy opens a new browser for each request, throttle beyond default limits for this URL
const res = await limiters.bypass.schedule(async () => bhttp.post(config.bypass.cloudflare.path, {
cmd: `request.${method}`,
url,
maxTimeout: options.timeout,
proxy: useProxy(url) ? {
url: `${config.proxy.host}:${config.proxy.port}`,
} : null,
}, {
encodeJSON: true,
}));
if (!res.statusCode === 200 || res.body?.status !== 'ok') {
throw new Error(`CloudFlare bypass failed for ${url} (${res.statusCode}): ${res.body?.message}`);
}
const resBody = extractJson(res.body.solution);
return {
body: resBody,
statusCode: res.body.solution.status,
headers: res.body.solution.headers,
};
}
async function request(method = 'get', url, body, requestOptions = {}, limiter) {
const http = requestOptions.session || bhttp;
@ -93,12 +141,17 @@ async function request(method = 'get', url, body, requestOptions = {}, limiter)
};
const withProxy = useProxy(url);
const withCloudflareBypass = options.bypassCloudflare && config.bypass.cloudflare.enable;
if (withProxy) {
options.agent = proxyAgent;
}
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}) ${url}`);
logger.debug(`${method.toUpperCase()} (${limiter._store.storeOptions.minTime}ms/${limiter._store.storeOptions.maxConcurrent}p${withProxy ? ' proxy' : ''}${withCloudflareBypass ? ' bypass' : ''}) ${url}`);
if (withCloudflareBypass) {
return bypassCloudflareRequest(url, method, body, options);
}
const res = await (body
? http[method](url, body, options)