Fixed session options in http module.

This commit is contained in:
DebaucheryLibrarian 2021-03-23 15:25:21 +01:00
parent b2ad031c54
commit 193af9bab5
7 changed files with 387 additions and 188 deletions

1
package-lock.json generated
View File

@ -28,6 +28,7 @@
"config": "^3.2.5",
"connect-session-knex": "^2.0.0",
"convert": "^1.6.2",
"cookie": "^0.4.0",
"csv-stringify": "^5.3.6",
"dayjs": "^1.8.21",
"dompurify": "^2.0.11",

View File

@ -89,6 +89,7 @@
"config": "^3.2.5",
"connect-session-knex": "^2.0.0",
"convert": "^1.6.2",
"cookie": "^0.4.0",
"csv-stringify": "^5.3.6",
"dayjs": "^1.8.21",
"dompurify": "^2.0.11",

View File

@ -795,438 +795,648 @@ const sites = [
// BANGBROS
{
name: 'Ass Parade',
url: 'https://bangbros.com/websites/assparade',
slug: 'assparade',
description: null,
alias: [
'ap',
],
url: 'https://bangbros.com/websites/assparade',
parent: 'bangbros',
parameters: { code: 'ap' },
parameters: {
code: 'ap',
},
},
{
name: 'AvaSpice',
url: 'https://bangbros.com/websites/avaspice',
slug: 'avaspice',
description: null,
alias: [
'av',
],
url: 'https://bangbros.com/websites/avaspice',
parent: 'bangbros',
parameters: { code: 'av' },
parameters: {
code: 'av',
},
},
{
name: 'Back Room Facials',
url: 'https://bangbros.com/websites/backroomfacials',
slug: 'backroomfacials',
description: null,
alias: [
'brf',
],
url: 'https://bangbros.com/websites/backroomfacials',
parent: 'bangbros',
parameters: { code: 'brf' },
parameters: {
code: 'brf',
},
},
{
name: 'Backroom MILF',
url: 'https://bangbros.com/websites/backroommilf',
slug: 'backroommilf',
description: null,
alias: [
'mf',
],
url: 'https://bangbros.com/websites/backroommilf',
parent: 'bangbros',
parameters: { code: 'mf' },
parameters: {
code: 'mf',
},
},
{
name: 'Ball Honeys',
url: 'https://bangbros.com/websites/ballhoneys',
slug: 'ballhoneys',
description: null,
alias: [
'es',
],
url: 'https://bangbros.com/websites/ballhoneys',
parent: 'bangbros',
parameters: { code: 'es' },
parameters: {
code: 'es',
},
},
{
name: 'BangBros 18',
url: 'https://bangbros.com/websites/bangbros18',
slug: 'bangbros18',
description: null,
alias: [
'bbe',
],
url: 'https://bangbros.com/websites/bangbros18',
parent: 'bangbros',
parameters: { code: 'bbe' },
parameters: {
code: 'bbe',
},
},
{
name: 'BangBros Angels',
url: 'https://bangbros.com/websites/bangbrosangels',
slug: 'bangbrosangels',
description: null,
alias: [
'bng',
],
url: 'https://bangbros.com/websites/bangbrosangels',
parent: 'bangbros',
parameters: { code: 'bng' },
parameters: {
code: 'bng',
},
},
{
name: 'Bangbros Clips',
url: 'https://bangbros.com/websites/bangbrosclips',
slug: 'bangbrosclips',
description: null,
alias: [
'bbc',
],
url: 'https://bangbros.com/websites/bangbrosclips',
parent: 'bangbros',
parameters: { code: 'bbc' },
parameters: {
code: 'bbc',
},
},
{
name: 'Bangbros Vault',
url: 'https://bangbros.com/websites/bangbrosvault',
slug: 'bangbrosvault',
description: null,
alias: [
'vault',
],
url: 'https://bangbros.com/websites/bangbrosvault',
parent: 'bangbros',
parameters: { code: 'vault' },
parameters: {
code: 'vault',
},
},
{
name: 'BangBros Remastered',
url: 'https://bangbros.com/websites/remaster',
slug: 'bangbrosremastered',
description: null,
alias: [
'rm',
],
url: 'https://bangbros.com/websites/remaster',
parent: 'bangbros',
parameters: { code: 'rm' },
parameters: {
code: 'rm',
},
},
{
name: 'Bang Bus',
url: 'https://bangbros.com/websites/bangbus',
slug: 'bangbus',
description: null,
alias: [
'bb',
],
url: 'https://bangbros.com/websites/bangbus',
parent: 'bangbros',
parameters: { code: 'bb' },
parameters: {
code: 'bb',
},
},
{
name: 'Bang Casting',
url: 'https://bangbros.com/websites/bangcasting',
slug: 'bangbroscasting',
description: null,
alias: [
'hih',
],
url: 'https://bangbros.com/websites/bangcasting',
parent: 'bangbros',
parameters: { code: 'hih' },
parameters: {
code: 'hih',
},
},
{
name: 'Bang POV',
url: 'https://bangbros.com/websites/bangpov',
slug: 'bangpov',
description: null,
alias: [
'bpov',
],
url: 'https://bangbros.com/websites/bangpov',
parent: 'bangbros',
parameters: { code: 'bpov' },
parameters: {
code: 'bpov',
},
},
{
name: 'Bang Tryouts',
url: 'https://bangbros.com/websites/bangtryouts',
slug: 'bangtryouts',
description: null,
alias: [
'bto',
],
url: 'https://bangbros.com/websites/bangtryouts',
parent: 'bangbros',
parameters: { code: 'bto' },
parameters: {
code: 'bto',
},
},
{
name: 'Big Mouthfuls',
url: 'https://bangbros.com/websites/bigmouthfuls',
slug: 'bigmouthfuls',
description: null,
alias: [
'bmf',
],
url: 'https://bangbros.com/websites/bigmouthfuls',
parent: 'bangbros',
parameters: { code: 'bmf' },
parameters: {
code: 'bmf',
},
},
{
name: 'Big Tit Cream Pie',
alias: ['btc'],
slug: 'bigtitcreampie',
alias: [
'btcp',
],
url: 'https://bangbros.com/websites/bigtitcreampie',
description: null,
parent: 'bangbros',
parameters: { code: 'btcp' },
parameters: {
code: 'btcp',
},
},
{
name: 'Big Tits, Round Asses',
url: 'https://bangbros.com/websites/bigtitsroundasses',
alias: ['btra'],
slug: 'bigtitsroundasses',
description: null,
alias: [
'btra',
],
url: 'https://bangbros.com/websites/bigtitsroundasses',
parent: 'bangbros',
parameters: { code: 'btra' },
parameters: {
code: 'btra',
},
},
{
name: 'BlowJob Fridays',
url: 'https://bangbros.com/websites/blowjobfridays',
slug: 'blowjobfridays',
description: null,
alias: [
'bj',
],
url: 'https://bangbros.com/websites/blowjobfridays',
parent: 'bangbros',
parameters: { code: 'bj' },
parameters: {
code: 'bj',
},
},
{
name: 'Blowjob Ninjas',
url: 'https://bangbros.com/websites/blowjobninjas',
slug: 'blowjobninjas',
description: null,
alias: [
'aa',
],
url: 'https://bangbros.com/websites/blowjobninjas',
parent: 'bangbros',
parameters: { code: 'aa' },
parameters: {
code: 'aa',
},
},
{
name: 'Boob Squad',
url: 'https://bangbros.com/websites/boobsquad',
slug: 'boobsquad',
description: null,
alias: [
'bs',
],
url: 'https://bangbros.com/websites/boobsquad',
parent: 'bangbros',
parameters: { code: 'bs' },
parameters: {
code: 'bs',
},
},
{
name: 'Brown Bunnies',
url: 'https://bangbros.com/websites/brownbunnies',
slug: 'brownbunnies',
description: null,
alias: [
'bkb',
],
url: 'https://bangbros.com/websites/brownbunnies',
parent: 'bangbros',
parameters: { code: 'bkb' },
parameters: {
code: 'bkb',
},
},
{
name: 'Can He Score?',
url: 'https://bangbros.com/websites/canhescore',
slug: 'canhescore',
description: null,
alias: [
'bd',
],
url: 'https://bangbros.com/websites/canhescore',
parent: 'bangbros',
parameters: { code: 'bd' },
parameters: {
code: 'bd',
},
},
{
name: 'Casting',
url: 'https://bangbros.com/websites/casting',
slug: 'casting',
description: null,
alias: [
'ca',
],
url: 'https://bangbros.com/websites/casting',
parent: 'bangbros',
parameters: { code: 'ca' },
parameters: {
code: 'ca',
},
},
{
name: 'Chongas',
url: 'https://bangbros.com/websites/chongas',
slug: 'chongas',
description: null,
alias: [
'ch',
],
url: 'https://bangbros.com/websites/chongas',
parent: 'bangbros',
parameters: { code: 'ch' },
parameters: {
code: 'ch',
},
},
{
name: 'Colombia Fuck Fest',
url: 'https://bangbros.com/websites/colombiafuckfest',
slug: 'colombiafuckfest',
description: null,
alias: [
'cff',
],
url: 'https://bangbros.com/websites/colombiafuckfest',
parent: 'bangbros',
parameters: { code: 'cff' },
parameters: {
code: 'cff',
},
},
{
name: 'Dirty World Tour',
url: 'https://bangbros.com/websites/dirtyworldtour',
slug: 'dirtyworldtour',
description: null,
alias: [
'bf',
],
url: 'https://bangbros.com/websites/dirtyworldtour',
parent: 'bangbros',
parameters: { code: 'bf' },
parameters: {
code: 'bf',
},
},
{
name: 'Dorm Invasion',
url: 'https://bangbros.com/websites/dorminvasion',
slug: 'dorminvasion',
description: null,
alias: [
'di',
],
url: 'https://bangbros.com/websites/dorminvasion',
parent: 'bangbros',
parameters: { code: 'di' },
parameters: {
code: 'di',
},
},
{
name: 'Facial Fest',
url: 'https://bangbros.com/websites/facialfest',
slug: 'facialfest',
description: null,
alias: [
'ff',
],
url: 'https://bangbros.com/websites/facialfest',
parent: 'bangbros',
parameters: { code: 'ff' },
parameters: {
code: 'ff',
},
},
{
name: 'Filthy Family',
url: 'https://filthyfamily.com',
slug: 'filthyfamily',
description: null,
url: 'https://filthyfamily.com',
parent: 'bangbros',
parameters: { legacy: true },
parameters: {
legacy: true,
},
},
{
name: 'Fuck Team Five',
url: 'https://bangbros.com/websites/fuckteamfive',
slug: 'fuckteamfive',
description: null,
alias: [
'bbw',
],
url: 'https://bangbros.com/websites/fuckteamfive',
parent: 'bangbros',
parameters: { code: 'bbw' },
parameters: {
code: 'bbw',
},
},
{
name: 'Glory Hole Loads',
url: 'https://bangbros.com/websites/gloryholeloads',
slug: 'gloryholeloads',
description: null,
alias: [
'ghl',
],
url: 'https://bangbros.com/websites/gloryholeloads',
parent: 'bangbros',
parameters: { code: 'ghl' },
parameters: {
code: 'ghl',
},
},
{
name: 'Latina Rampage',
url: 'https://bangbros.com/websites/latinarampage',
slug: 'latinarampage',
description: null,
alias: [
'lrp',
],
url: 'https://bangbros.com/websites/latinarampage',
parent: 'bangbros',
parameters: { code: 'lrp' },
parameters: {
code: 'lrp',
},
},
{
name: 'Living With Anna',
url: 'https://bangbros.com/websites/livingwithanna',
slug: 'livingwithanna',
description: null,
alias: [
'lr',
],
url: 'https://bangbros.com/websites/livingwithanna',
parent: 'bangbros',
parameters: { code: 'lr' },
parameters: {
code: 'lr',
},
},
{
name: 'Magical Feet',
url: 'https://bangbros.com/websites/magicalfeet',
slug: 'magicalfeet',
description: null,
alias: [
'fj',
],
url: 'https://bangbros.com/websites/magicalfeet',
parent: 'bangbros',
parameters: { code: 'fj' },
parameters: {
code: 'fj',
},
},
{
name: 'Milf Soup',
url: 'https://bangbros.com/websites/milfsoup',
slug: 'milfsoup',
description: null,
alias: [
'ms',
],
url: 'https://bangbros.com/websites/milfsoup',
parent: 'bangbros',
parameters: { code: 'ms' },
parameters: {
code: 'ms',
},
},
{
name: 'MomIsHorny',
url: 'https://bangbros.com/websites/momishorny',
slug: 'momishorny',
description: null,
alias: [
'mih',
],
url: 'https://bangbros.com/websites/momishorny',
parent: 'bangbros',
parameters: { code: 'mih' },
parameters: {
code: 'mih',
},
},
{
name: 'Monsters of Cock',
url: 'https://bangbros.com/websites/monstersofcock',
slug: 'monstersofcock',
description: null,
alias: [
'mc',
],
url: 'https://bangbros.com/websites/monstersofcock',
parent: 'bangbros',
parameters: { code: 'mc' },
parameters: {
code: 'mc',
},
},
{
name: 'Mr CamelToe',
url: 'https://bangbros.com/websites/mrcameltoe',
slug: 'mrcameltoe',
description: null,
alias: [
'ct',
],
url: 'https://bangbros.com/websites/mrcameltoe',
parent: 'bangbros',
parameters: { code: 'ct' },
parameters: {
code: 'ct',
},
},
{
name: 'My Dirty Maid',
slug: 'mydirtymaid',
alias: ['mdm'],
alias: [
'mda',
],
url: 'https://bangbros.com/websites/mydirtymaid',
description: null,
parent: 'bangbros',
parameters: { code: 'mda' },
parameters: {
code: 'mda',
},
},
{
name: 'My Life In Brazil',
url: 'https://bangbros.com/websites/mylifeinbrazil',
slug: 'mylifeinbrazil',
description: null,
alias: [
'mb',
],
url: 'https://bangbros.com/websites/mylifeinbrazil',
parent: 'bangbros',
parameters: { code: 'mb' },
parameters: {
code: 'mb',
},
},
{
name: 'Newbie Black',
url: 'https://bangbros.com/websites/newbieblack',
slug: 'newbieblack',
description: null,
alias: [
'blkg',
],
url: 'https://bangbros.com/websites/newbieblack',
parent: 'bangbros',
parameters: { code: 'blkg' },
parameters: {
code: 'blkg',
},
},
{
name: 'Party of Three',
url: 'https://bangbros.com/websites/partyofthree',
slug: 'partyofthree',
description: null,
alias: [
'ls',
],
url: 'https://bangbros.com/websites/partyofthree',
parent: 'bangbros',
parameters: { code: 'ls' },
parameters: {
code: 'ls',
},
},
{
name: 'Pawg',
url: 'https://bangbros.com/websites/pawg',
slug: 'pawg',
description: null,
alias: [
'pwg',
],
url: 'https://bangbros.com/websites/pawg',
parent: 'bangbros',
parameters: { code: 'pwg' },
parameters: {
code: 'pwg',
},
},
{
name: 'Penny Show',
url: 'https://bangbros.com/websites/pennyshow',
slug: 'pennyshow',
description: null,
alias: [
'ps',
],
url: 'https://bangbros.com/websites/pennyshow',
parent: 'bangbros',
parameters: { code: 'ps' },
parameters: {
code: 'ps',
},
},
{
name: 'Porn Star Spa',
url: 'https://bangbros.com/websites/pornstarspa',
slug: 'pornstarspa',
description: null,
alias: [
'pos',
],
url: 'https://bangbros.com/websites/pornstarspa',
parent: 'bangbros',
parameters: { code: 'pos' },
parameters: {
code: 'pos',
},
},
{
name: 'Power Munch',
url: 'https://bangbros.com/websites/powermunch',
slug: 'powermunch',
description: null,
alias: [
'pm',
],
url: 'https://bangbros.com/websites/powermunch',
parent: 'bangbros',
parameters: { code: 'pm' },
parameters: {
code: 'pm',
},
},
{
name: 'Public Bang',
url: 'https://bangbros.com/websites/publicbang',
slug: 'publicbang',
description: null,
alias: [
'pb',
],
url: 'https://bangbros.com/websites/publicbang',
parent: 'bangbros',
parameters: { code: 'pb' },
parameters: {
code: 'pb',
},
},
{
name: 'Slutty White Girls',
url: 'https://bangbros.com/websites/sluttywhitegirls',
slug: 'sluttywhitegirls',
description: null,
alias: [
'swg',
],
url: 'https://bangbros.com/websites/sluttywhitegirls',
parent: 'bangbros',
parameters: { code: 'swg' },
parameters: {
code: 'swg',
},
},
{
name: 'Stepmom Videos',
url: 'https://bangbros.com/websites/stepmomvideos',
slug: 'stepmomvideos',
description: null,
alias: [
'smv',
],
url: 'https://bangbros.com/websites/stepmomvideos',
parent: 'bangbros',
parameters: { code: 'smv' },
parameters: {
code: 'smv',
},
},
{
name: 'Street Ranger',
url: 'https://bangbros.com/websites/thewheeler',
slug: 'streetranger',
description: null,
alias: [
'sg',
],
url: 'https://bangbros.com/websites/thewheeler',
parent: 'bangbros',
parameters: { code: 'sg' },
parameters: {
code: 'sg',
},
},
{
name: 'Tugjobs',
url: 'https://bangbros.com/websites/tugjobs',
slug: 'tugjobs',
description: null,
alias: [
'hj',
],
url: 'https://bangbros.com/websites/tugjobs',
parent: 'bangbros',
parameters: { code: 'hj' },
parameters: {
code: 'hj',
},
},
{
name: 'Working Latinas',
url: 'https://bangbros.com/websites/workinglatinas',
slug: 'workinglatinas',
description: null,
alias: [
'lw',
],
url: 'https://bangbros.com/websites/workinglatinas',
parent: 'bangbros',
parameters: { code: 'lw' },
parameters: {
code: 'lw',
},
},
{
name: 'MILF Lessons',
url: 'https://bangbros.com/websites/milflessons',
slug: 'milflessons',
description: null,
alias: [
'ml',
],
url: 'https://bangbros.com/websites/milflessons',
parent: 'bangbros',
parameters: { code: 'ml' },
parameters: {
code: 'ml',
},
},
{
name: 'Mr. Anal',
url: 'https://bangbros.com/websites/mranal',
slug: 'mranal',
description: null,
alias: [
'ma',
],
url: 'https://bangbros.com/websites/mranal',
parent: 'bangbros',
parameters: { code: 'ma' },
parameters: {
code: 'ma',
},
},
// BLOWPASS
{

View File

@ -3,13 +3,13 @@
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const cookie = require('cookie');
const moment = require('moment');
const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
const http = require('../utils/http');
const { inchesToCm, lbsToKg } = require('../utils/convert');
const { cookieToData } = require('../utils/cookies');
function getThumbs(scene) {
if (scene.images.poster) {
@ -174,7 +174,7 @@ async function getSession(site, parameters) {
if (res.statusCode === 200) {
const cookieString = await cookieJar.getCookieStringAsync(sessionUrl);
const { instance_token: instanceToken } = cookieToData(cookieString);
const { instance_token: instanceToken } = cookie.parse(cookieString);
return { session, instanceToken };
}
@ -254,11 +254,11 @@ async function fetchLatest(site, page = 1, options) {
},
});
if (res.statusCode === 200 && res.body.result) {
if (res.status === 200 && res.body.result) {
return scrapeLatest(res.body.result, site);
}
return null;
return res.statusCode;
}
async function fetchUpcoming(site, page, options) {
@ -282,7 +282,7 @@ async function fetchUpcoming(site, page, options) {
return scrapeLatest(res.body.result, site, true);
}
return null;
return res.statusCode;
}
async function fetchScene(url, site, baseScene, options) {
@ -303,7 +303,7 @@ async function fetchScene(url, site, baseScene, options) {
},
});
if (res.statusCode === 200 && res.body.result) {
if (res.status === 200 && res.body.result) {
return scrapeScene(res.body.result, url, site);
}

View File

@ -227,7 +227,7 @@ async function updateReleasesSearch(releaseIds) {
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||

View File

@ -1,16 +0,0 @@
'use strict';
function cookieToData(cookieString) {
return cookieString.split('; ').reduce((acc, cookie) => {
const [key, value] = cookie.split('=');
return {
...acc,
[key]: value,
};
}, {});
}
module.exports = {
cookieToData,
};

View File

@ -82,7 +82,11 @@ function getLimiter(options = {}, url) {
async function request(method = 'get', url, body, requestOptions = {}, limiter) {
const http = requestOptions.session || bhttp;
const options = requestOptions;
const options = {
...requestOptions,
session: null,
};
const withProxy = useProxy(url);
@ -151,7 +155,6 @@ async function scheduleRequest(method = 'get', url, body, requestOptions = {}) {
},
responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout,
stream: !!requestOptions.destination,
session: null,
};
const limiter = getLimiter(options, url);