From 193af9bab5cfb119dc21202c7f025eab22a147f7 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 23 Mar 2021 15:25:21 +0100 Subject: [PATCH] Fixed session options in http module. --- package-lock.json | 1 + package.json | 1 + seeds/02_sites.js | 536 +++++++++++++++++++++++++++------------ src/scrapers/mindgeek.js | 12 +- src/store-releases.js | 2 +- src/utils/cookies.js | 16 -- src/utils/http.js | 7 +- 7 files changed, 387 insertions(+), 188 deletions(-) delete mode 100644 src/utils/cookies.js diff --git a/package-lock.json b/package-lock.json index bf7d9e53..4431473e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "config": "^3.2.5", "connect-session-knex": "^2.0.0", "convert": "^1.6.2", + "cookie": "^0.4.0", "csv-stringify": "^5.3.6", "dayjs": "^1.8.21", "dompurify": "^2.0.11", diff --git a/package.json b/package.json index 4b58512b..98744de0 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "config": "^3.2.5", "connect-session-knex": "^2.0.0", "convert": "^1.6.2", + "cookie": "^0.4.0", "csv-stringify": "^5.3.6", "dayjs": "^1.8.21", "dompurify": "^2.0.11", diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 2c648e16..9cc69699 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -795,438 +795,648 @@ const sites = [ // BANGBROS { name: 'Ass Parade', - url: 'https://bangbros.com/websites/assparade', slug: 'assparade', - description: null, + alias: [ + 'ap', + ], + url: 'https://bangbros.com/websites/assparade', parent: 'bangbros', - parameters: { code: 'ap' }, + parameters: { + code: 'ap', + }, }, { name: 'AvaSpice', - url: 'https://bangbros.com/websites/avaspice', slug: 'avaspice', - description: null, + alias: [ + 'av', + ], + url: 'https://bangbros.com/websites/avaspice', parent: 'bangbros', - parameters: { code: 'av' }, + parameters: { + code: 'av', + }, }, { name: 'Back Room Facials', - url: 'https://bangbros.com/websites/backroomfacials', slug: 'backroomfacials', - description: null, + alias: [ + 'brf', + ], + url: 'https://bangbros.com/websites/backroomfacials', parent: 'bangbros', - parameters: { code: 'brf' }, + parameters: { + code: 'brf', + }, }, { name: 'Backroom MILF', - url: 'https://bangbros.com/websites/backroommilf', slug: 'backroommilf', - description: null, + alias: [ + 'mf', + ], + url: 'https://bangbros.com/websites/backroommilf', parent: 'bangbros', - parameters: { code: 'mf' }, + parameters: { + code: 'mf', + }, }, { name: 'Ball Honeys', - url: 'https://bangbros.com/websites/ballhoneys', slug: 'ballhoneys', - description: null, + alias: [ + 'es', + ], + url: 'https://bangbros.com/websites/ballhoneys', parent: 'bangbros', - parameters: { code: 'es' }, + parameters: { + code: 'es', + }, }, { name: 'BangBros 18', - url: 'https://bangbros.com/websites/bangbros18', slug: 'bangbros18', - description: null, + alias: [ + 'bbe', + ], + url: 'https://bangbros.com/websites/bangbros18', parent: 'bangbros', - parameters: { code: 'bbe' }, + parameters: { + code: 'bbe', + }, }, { name: 'BangBros Angels', - url: 'https://bangbros.com/websites/bangbrosangels', slug: 'bangbrosangels', - description: null, + alias: [ + 'bng', + ], + url: 'https://bangbros.com/websites/bangbrosangels', parent: 'bangbros', - parameters: { code: 'bng' }, + parameters: { + code: 'bng', + }, }, { name: 'Bangbros Clips', - url: 'https://bangbros.com/websites/bangbrosclips', slug: 'bangbrosclips', - description: null, + alias: [ + 'bbc', + ], + url: 'https://bangbros.com/websites/bangbrosclips', parent: 'bangbros', - parameters: { code: 'bbc' }, + parameters: { + code: 'bbc', + }, }, { name: 'Bangbros Vault', - url: 'https://bangbros.com/websites/bangbrosvault', slug: 'bangbrosvault', - description: null, + alias: [ + 'vault', + ], + url: 'https://bangbros.com/websites/bangbrosvault', parent: 'bangbros', - parameters: { code: 'vault' }, + parameters: { + code: 'vault', + }, }, { name: 'BangBros Remastered', - url: 'https://bangbros.com/websites/remaster', slug: 'bangbrosremastered', - description: null, + alias: [ + 'rm', + ], + url: 'https://bangbros.com/websites/remaster', parent: 'bangbros', - parameters: { code: 'rm' }, + parameters: { + code: 'rm', + }, }, { name: 'Bang Bus', - url: 'https://bangbros.com/websites/bangbus', slug: 'bangbus', - description: null, + alias: [ + 'bb', + ], + url: 'https://bangbros.com/websites/bangbus', parent: 'bangbros', - parameters: { code: 'bb' }, + parameters: { + code: 'bb', + }, }, { name: 'Bang Casting', - url: 'https://bangbros.com/websites/bangcasting', slug: 'bangbroscasting', - description: null, + alias: [ + 'hih', + ], + url: 'https://bangbros.com/websites/bangcasting', parent: 'bangbros', - parameters: { code: 'hih' }, + parameters: { + code: 'hih', + }, }, { name: 'Bang POV', - url: 'https://bangbros.com/websites/bangpov', slug: 'bangpov', - description: null, + alias: [ + 'bpov', + ], + url: 'https://bangbros.com/websites/bangpov', parent: 'bangbros', - parameters: { code: 'bpov' }, + parameters: { + code: 'bpov', + }, }, { name: 'Bang Tryouts', - url: 'https://bangbros.com/websites/bangtryouts', slug: 'bangtryouts', - description: null, + alias: [ + 'bto', + ], + url: 'https://bangbros.com/websites/bangtryouts', parent: 'bangbros', - parameters: { code: 'bto' }, + parameters: { + code: 'bto', + }, }, { name: 'Big Mouthfuls', - url: 'https://bangbros.com/websites/bigmouthfuls', slug: 'bigmouthfuls', - description: null, + alias: [ + 'bmf', + ], + url: 'https://bangbros.com/websites/bigmouthfuls', parent: 'bangbros', - parameters: { code: 'bmf' }, + parameters: { + code: 'bmf', + }, }, { name: 'Big Tit Cream Pie', - alias: ['btc'], slug: 'bigtitcreampie', + alias: [ + 'btcp', + ], url: 'https://bangbros.com/websites/bigtitcreampie', - description: null, parent: 'bangbros', - parameters: { code: 'btcp' }, + parameters: { + code: 'btcp', + }, }, { name: 'Big Tits, Round Asses', - url: 'https://bangbros.com/websites/bigtitsroundasses', - alias: ['btra'], slug: 'bigtitsroundasses', - description: null, + alias: [ + 'btra', + ], + url: 'https://bangbros.com/websites/bigtitsroundasses', parent: 'bangbros', - parameters: { code: 'btra' }, + parameters: { + code: 'btra', + }, }, { name: 'BlowJob Fridays', - url: 'https://bangbros.com/websites/blowjobfridays', slug: 'blowjobfridays', - description: null, + alias: [ + 'bj', + ], + url: 'https://bangbros.com/websites/blowjobfridays', parent: 'bangbros', - parameters: { code: 'bj' }, + parameters: { + code: 'bj', + }, }, { name: 'Blowjob Ninjas', - url: 'https://bangbros.com/websites/blowjobninjas', slug: 'blowjobninjas', - description: null, + alias: [ + 'aa', + ], + url: 'https://bangbros.com/websites/blowjobninjas', parent: 'bangbros', - parameters: { code: 'aa' }, + parameters: { + code: 'aa', + }, }, { name: 'Boob Squad', - url: 'https://bangbros.com/websites/boobsquad', slug: 'boobsquad', - description: null, + alias: [ + 'bs', + ], + url: 'https://bangbros.com/websites/boobsquad', parent: 'bangbros', - parameters: { code: 'bs' }, + parameters: { + code: 'bs', + }, }, { name: 'Brown Bunnies', - url: 'https://bangbros.com/websites/brownbunnies', slug: 'brownbunnies', - description: null, + alias: [ + 'bkb', + ], + url: 'https://bangbros.com/websites/brownbunnies', parent: 'bangbros', - parameters: { code: 'bkb' }, + parameters: { + code: 'bkb', + }, }, { name: 'Can He Score?', - url: 'https://bangbros.com/websites/canhescore', slug: 'canhescore', - description: null, + alias: [ + 'bd', + ], + url: 'https://bangbros.com/websites/canhescore', parent: 'bangbros', - parameters: { code: 'bd' }, + parameters: { + code: 'bd', + }, }, { name: 'Casting', - url: 'https://bangbros.com/websites/casting', slug: 'casting', - description: null, + alias: [ + 'ca', + ], + url: 'https://bangbros.com/websites/casting', parent: 'bangbros', - parameters: { code: 'ca' }, + parameters: { + code: 'ca', + }, }, { name: 'Chongas', - url: 'https://bangbros.com/websites/chongas', slug: 'chongas', - description: null, + alias: [ + 'ch', + ], + url: 'https://bangbros.com/websites/chongas', parent: 'bangbros', - parameters: { code: 'ch' }, + parameters: { + code: 'ch', + }, }, { name: 'Colombia Fuck Fest', - url: 'https://bangbros.com/websites/colombiafuckfest', slug: 'colombiafuckfest', - description: null, + alias: [ + 'cff', + ], + url: 'https://bangbros.com/websites/colombiafuckfest', parent: 'bangbros', - parameters: { code: 'cff' }, + parameters: { + code: 'cff', + }, }, { name: 'Dirty World Tour', - url: 'https://bangbros.com/websites/dirtyworldtour', slug: 'dirtyworldtour', - description: null, + alias: [ + 'bf', + ], + url: 'https://bangbros.com/websites/dirtyworldtour', parent: 'bangbros', - parameters: { code: 'bf' }, + parameters: { + code: 'bf', + }, }, { name: 'Dorm Invasion', - url: 'https://bangbros.com/websites/dorminvasion', slug: 'dorminvasion', - description: null, + alias: [ + 'di', + ], + url: 'https://bangbros.com/websites/dorminvasion', parent: 'bangbros', - parameters: { code: 'di' }, + parameters: { + code: 'di', + }, }, { name: 'Facial Fest', - url: 'https://bangbros.com/websites/facialfest', slug: 'facialfest', - description: null, + alias: [ + 'ff', + ], + url: 'https://bangbros.com/websites/facialfest', parent: 'bangbros', - parameters: { code: 'ff' }, + parameters: { + code: 'ff', + }, }, { name: 'Filthy Family', - url: 'https://filthyfamily.com', slug: 'filthyfamily', - description: null, + url: 'https://filthyfamily.com', parent: 'bangbros', - parameters: { legacy: true }, + parameters: { + legacy: true, + }, }, { name: 'Fuck Team Five', - url: 'https://bangbros.com/websites/fuckteamfive', slug: 'fuckteamfive', - description: null, + alias: [ + 'bbw', + ], + url: 'https://bangbros.com/websites/fuckteamfive', parent: 'bangbros', - parameters: { code: 'bbw' }, + parameters: { + code: 'bbw', + }, }, { name: 'Glory Hole Loads', - url: 'https://bangbros.com/websites/gloryholeloads', slug: 'gloryholeloads', - description: null, + alias: [ + 'ghl', + ], + url: 'https://bangbros.com/websites/gloryholeloads', parent: 'bangbros', - parameters: { code: 'ghl' }, + parameters: { + code: 'ghl', + }, }, { name: 'Latina Rampage', - url: 'https://bangbros.com/websites/latinarampage', slug: 'latinarampage', - description: null, + alias: [ + 'lrp', + ], + url: 'https://bangbros.com/websites/latinarampage', parent: 'bangbros', - parameters: { code: 'lrp' }, + parameters: { + code: 'lrp', + }, }, { name: 'Living With Anna', - url: 'https://bangbros.com/websites/livingwithanna', slug: 'livingwithanna', - description: null, + alias: [ + 'lr', + ], + url: 'https://bangbros.com/websites/livingwithanna', parent: 'bangbros', - parameters: { code: 'lr' }, + parameters: { + code: 'lr', + }, }, { name: 'Magical Feet', - url: 'https://bangbros.com/websites/magicalfeet', slug: 'magicalfeet', - description: null, + alias: [ + 'fj', + ], + url: 'https://bangbros.com/websites/magicalfeet', parent: 'bangbros', - parameters: { code: 'fj' }, + parameters: { + code: 'fj', + }, }, { name: 'Milf Soup', - url: 'https://bangbros.com/websites/milfsoup', slug: 'milfsoup', - description: null, + alias: [ + 'ms', + ], + url: 'https://bangbros.com/websites/milfsoup', parent: 'bangbros', - parameters: { code: 'ms' }, + parameters: { + code: 'ms', + }, }, { name: 'MomIsHorny', - url: 'https://bangbros.com/websites/momishorny', slug: 'momishorny', - description: null, + alias: [ + 'mih', + ], + url: 'https://bangbros.com/websites/momishorny', parent: 'bangbros', - parameters: { code: 'mih' }, + parameters: { + code: 'mih', + }, }, { name: 'Monsters of Cock', - url: 'https://bangbros.com/websites/monstersofcock', slug: 'monstersofcock', - description: null, + alias: [ + 'mc', + ], + url: 'https://bangbros.com/websites/monstersofcock', parent: 'bangbros', - parameters: { code: 'mc' }, + parameters: { + code: 'mc', + }, }, { name: 'Mr CamelToe', - url: 'https://bangbros.com/websites/mrcameltoe', slug: 'mrcameltoe', - description: null, + alias: [ + 'ct', + ], + url: 'https://bangbros.com/websites/mrcameltoe', parent: 'bangbros', - parameters: { code: 'ct' }, + parameters: { + code: 'ct', + }, }, { name: 'My Dirty Maid', slug: 'mydirtymaid', - alias: ['mdm'], + alias: [ + 'mda', + ], url: 'https://bangbros.com/websites/mydirtymaid', - description: null, parent: 'bangbros', - parameters: { code: 'mda' }, + parameters: { + code: 'mda', + }, }, { name: 'My Life In Brazil', - url: 'https://bangbros.com/websites/mylifeinbrazil', slug: 'mylifeinbrazil', - description: null, + alias: [ + 'mb', + ], + url: 'https://bangbros.com/websites/mylifeinbrazil', parent: 'bangbros', - parameters: { code: 'mb' }, + parameters: { + code: 'mb', + }, }, { name: 'Newbie Black', - url: 'https://bangbros.com/websites/newbieblack', slug: 'newbieblack', - description: null, + alias: [ + 'blkg', + ], + url: 'https://bangbros.com/websites/newbieblack', parent: 'bangbros', - parameters: { code: 'blkg' }, + parameters: { + code: 'blkg', + }, }, { name: 'Party of Three', - url: 'https://bangbros.com/websites/partyofthree', slug: 'partyofthree', - description: null, + alias: [ + 'ls', + ], + url: 'https://bangbros.com/websites/partyofthree', parent: 'bangbros', - parameters: { code: 'ls' }, + parameters: { + code: 'ls', + }, }, { name: 'Pawg', - url: 'https://bangbros.com/websites/pawg', slug: 'pawg', - description: null, + alias: [ + 'pwg', + ], + url: 'https://bangbros.com/websites/pawg', parent: 'bangbros', - parameters: { code: 'pwg' }, + parameters: { + code: 'pwg', + }, }, { name: 'Penny Show', - url: 'https://bangbros.com/websites/pennyshow', slug: 'pennyshow', - description: null, + alias: [ + 'ps', + ], + url: 'https://bangbros.com/websites/pennyshow', parent: 'bangbros', - parameters: { code: 'ps' }, + parameters: { + code: 'ps', + }, }, { name: 'Porn Star Spa', - url: 'https://bangbros.com/websites/pornstarspa', slug: 'pornstarspa', - description: null, + alias: [ + 'pos', + ], + url: 'https://bangbros.com/websites/pornstarspa', parent: 'bangbros', - parameters: { code: 'pos' }, + parameters: { + code: 'pos', + }, }, { name: 'Power Munch', - url: 'https://bangbros.com/websites/powermunch', slug: 'powermunch', - description: null, + alias: [ + 'pm', + ], + url: 'https://bangbros.com/websites/powermunch', parent: 'bangbros', - parameters: { code: 'pm' }, + parameters: { + code: 'pm', + }, }, { name: 'Public Bang', - url: 'https://bangbros.com/websites/publicbang', slug: 'publicbang', - description: null, + alias: [ + 'pb', + ], + url: 'https://bangbros.com/websites/publicbang', parent: 'bangbros', - parameters: { code: 'pb' }, + parameters: { + code: 'pb', + }, }, { name: 'Slutty White Girls', - url: 'https://bangbros.com/websites/sluttywhitegirls', slug: 'sluttywhitegirls', - description: null, + alias: [ + 'swg', + ], + url: 'https://bangbros.com/websites/sluttywhitegirls', parent: 'bangbros', - parameters: { code: 'swg' }, + parameters: { + code: 'swg', + }, }, { name: 'Stepmom Videos', - url: 'https://bangbros.com/websites/stepmomvideos', slug: 'stepmomvideos', - description: null, + alias: [ + 'smv', + ], + url: 'https://bangbros.com/websites/stepmomvideos', parent: 'bangbros', - parameters: { code: 'smv' }, + parameters: { + code: 'smv', + }, }, { name: 'Street Ranger', - url: 'https://bangbros.com/websites/thewheeler', slug: 'streetranger', - description: null, + alias: [ + 'sg', + ], + url: 'https://bangbros.com/websites/thewheeler', parent: 'bangbros', - parameters: { code: 'sg' }, + parameters: { + code: 'sg', + }, }, { name: 'Tugjobs', - url: 'https://bangbros.com/websites/tugjobs', slug: 'tugjobs', - description: null, + alias: [ + 'hj', + ], + url: 'https://bangbros.com/websites/tugjobs', parent: 'bangbros', - parameters: { code: 'hj' }, + parameters: { + code: 'hj', + }, }, { name: 'Working Latinas', - url: 'https://bangbros.com/websites/workinglatinas', slug: 'workinglatinas', - description: null, + alias: [ + 'lw', + ], + url: 'https://bangbros.com/websites/workinglatinas', parent: 'bangbros', - parameters: { code: 'lw' }, + parameters: { + code: 'lw', + }, }, { name: 'MILF Lessons', - url: 'https://bangbros.com/websites/milflessons', slug: 'milflessons', - description: null, + alias: [ + 'ml', + ], + url: 'https://bangbros.com/websites/milflessons', parent: 'bangbros', - parameters: { code: 'ml' }, + parameters: { + code: 'ml', + }, }, { name: 'Mr. Anal', - url: 'https://bangbros.com/websites/mranal', slug: 'mranal', - description: null, + alias: [ + 'ma', + ], + url: 'https://bangbros.com/websites/mranal', parent: 'bangbros', - parameters: { code: 'ma' }, + parameters: { + code: 'ma', + }, }, // BLOWPASS { diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index 7c40c778..3e646b1a 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -3,13 +3,13 @@ /* eslint-disable newline-per-chained-call */ const Promise = require('bluebird'); const { CookieJar } = Promise.promisifyAll(require('tough-cookie')); +const cookie = require('cookie'); const moment = require('moment'); const qu = require('../utils/qu'); const slugify = require('../utils/slugify'); const http = require('../utils/http'); const { inchesToCm, lbsToKg } = require('../utils/convert'); -const { cookieToData } = require('../utils/cookies'); function getThumbs(scene) { if (scene.images.poster) { @@ -174,7 +174,7 @@ async function getSession(site, parameters) { if (res.statusCode === 200) { const cookieString = await cookieJar.getCookieStringAsync(sessionUrl); - const { instance_token: instanceToken } = cookieToData(cookieString); + const { instance_token: instanceToken } = cookie.parse(cookieString); return { session, instanceToken }; } @@ -254,11 +254,11 @@ async function fetchLatest(site, page = 1, options) { }, }); - if (res.statusCode === 200 && res.body.result) { + if (res.status === 200 && res.body.result) { return scrapeLatest(res.body.result, site); } - return null; + return res.statusCode; } async function fetchUpcoming(site, page, options) { @@ -282,7 +282,7 @@ async function fetchUpcoming(site, page, options) { return scrapeLatest(res.body.result, site, true); } - return null; + return res.statusCode; } async function fetchScene(url, site, baseScene, options) { @@ -303,7 +303,7 @@ async function fetchScene(url, site, baseScene, options) { }, }); - if (res.statusCode === 200 && res.body.result) { + if (res.status === 200 && res.body.result) { return scrapeScene(res.body.result, url, site); } diff --git a/src/store-releases.js b/src/store-releases.js index 73f60d38..1211d2dc 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -227,7 +227,7 @@ async function updateReleasesSearch(releaseIds) { COALESCE(parents.slug, '') || ' ' || COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' || - COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' || + COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' || STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' || diff --git a/src/utils/cookies.js b/src/utils/cookies.js deleted file mode 100644 index d14e28e6..00000000 --- a/src/utils/cookies.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict'; - -function cookieToData(cookieString) { - return cookieString.split('; ').reduce((acc, cookie) => { - const [key, value] = cookie.split('='); - - return { - ...acc, - [key]: value, - }; - }, {}); -} - -module.exports = { - cookieToData, -}; diff --git a/src/utils/http.js b/src/utils/http.js index 99856ffc..20ea82df 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -82,7 +82,11 @@ function getLimiter(options = {}, url) { async function request(method = 'get', url, body, requestOptions = {}, limiter) { const http = requestOptions.session || bhttp; - const options = requestOptions; + + const options = { + ...requestOptions, + session: null, + }; const withProxy = useProxy(url); @@ -151,7 +155,6 @@ async function scheduleRequest(method = 'get', url, body, requestOptions = {}) { }, responseTimeout: requestOptions.responseTimeout || requestOptions.timeout || defaultOptions.timeout, stream: !!requestOptions.destination, - session: null, }; const limiter = getLimiter(options, url);