DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added Brazzers scraper. Removed scene-only sites in database in favor of automatic network fallback.

This commit is contained in:
ThePendulum 2019-04-04 20:22:47 +02:00
parent 784542253b
commit 6190247ac5
9 changed files with 494 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
config/default.js
View File

@ -2,6 +2,41 @@
module.exports = {
include: [
['brazzers', [
'assesinpublic',
'babygotboobs',
'bigbuttslikeitbig',
'bigtitsatschool',
'bigtitsatwork',
'bigtitsinsports',
'bigtitsinuniform',
'bigwetbutts',
'brazzersenespanol',
'brazzersexxtra',
'brazzerslive',
'brazzersvault',
'bustyandreal',
'bustyz',
'buttsandblacks',
'cfnm',
'daywithapornstar',
'dirtymasseur',
'doctoradventures',
'hotandmean',
'hotchicksbigasses',
'jugfuckers',
'milfslikeitbig',
'mommygotboobs',
'momsincontrol',
'pornstarslikeitbig',
'racksandblacks',
'realwifestories',
'sexproadventures',
'shesgonnasquirt',
'teenslikeitbig',
'teenslikeitblack',
'zzseries',
]],
'julesjordan',
['kink', [
'boundgangbangs',

6
seeds/networks.js
View File

@ -4,6 +4,12 @@
exports.seed = knex => Promise.resolve()
.then(() => knex('networks').del())
.then(() => knex('networks').insert([
{
id: 'brazzers',
name: 'Brazzers',
url: 'https://www.brazzers.com',
description: 'Brazzers homepage is updated daily with official HD porn scenes. Our hottest videos and sex series are filled with big tits, sexy milf, top pornstars and special events.',
},
{
id: 'julesjordan',
name: 'Jules Jordan',

283
seeds/sites.js
View File

@ -4,6 +4,271 @@
exports.seed = knex => Promise.resolve()
.then(() => knex('sites').del())
.then(() => knex('sites').insert([
// BRAZZERS
{
id: 'momsincontrol',
name: 'Moms in Control',
label: 'MIC',
url: 'https://www.brazzers.com/sites/view/id/155/moms-in-control',
description: "There's nothing hotter than seeing a wholesome MILf get dirty, and that's exactly what MILFs in Control is all about: the hottest, sluttiest cougars in the business taking control of sexy situations to get exactly what they want. Feast your eyes as these mature beauties suck and fuck huge cocks, dominating big-dick studs and hot teen sluts until they get the cum that all MILFs crave!",
network_id: 'brazzers',
},
{
id: 'pornstarslikeitbig',
name: 'Pornstars Like it Big',
label: 'PLIB',
url: 'https://www.brazzers.com/sites/view/id/24/pornstars-like-it-big',
description: "A real big dick, that's what everyone wants. Porn-stars are no exception, all the biggest stars agree; BIG COCK is for them. Check out how it stretches their tiny pussies and cums on their round tits. We've got the best chicks jocking the biggest dicks.",
network_id: 'brazzers',
},
{
id: 'bigtitsatwork',
name: 'Big Tits at Work',
label: 'BTAW',
url: 'https://www.brazzers.com/sites/view/id/15/big-tits-at-work',
description: 'Sitting at your desk, wishing you can fuck every busty coworker you have? Well, stop dreaming and step inside Big Tits At Work where you can watch real life work adventures caught on tape. Nothing But Big Breasted Work Professionals getting drilled all day long...',
network_id: 'brazzers',
},
{
id: 'bigtitsatschool',
name: 'Big Tits at School',
label: 'BTAS',
url: 'https://www.brazzers.com/sites/view/id/20/big-tits-at-school',
description: "The windows have been fogging up at Big Tits At School. Just take a peek inside one of our classrooms and you'll see our smoking hot busty students and big boobed dominant teachers getting their wet pussies stuffed with cock. Stay in your seat! you haven't been dismissed yet.",
network_id: 'brazzers',
},
{
id: 'babygotboobs',
name: 'Baby Got Boobs',
label: 'BGB',
url: 'https://www.brazzers.com/sites/view/id/9/baby-got-boobs',
description: "From fresh-faced teen to total slut, baby has boobs and she isn't afraid to show them. But does she know how to use them? These teens crave monster cock in their tight pussies, whether they're ready for a big dicking is another matter.",
network_id: 'brazzers',
},
{
id: 'realwifestories',
name: 'Real Wife Stories',
label: 'RWS',
url: 'https://www.brazzers.com/sites/view/id/52/real-wife-stories',
description: "You might bring home the bacon, but your wife is still starving. That slut is hungry for cock, she can't get enough, and if you starve her any more she'll get it wherever she can. Better leave work early, or your big-titted wife might just have some giant cock getting squeezed into her waiting pussy, and it won't be yours.",
network_id: 'brazzers',
},
{
id: 'teenslikeitbig',
name: 'Teens Like It Big',
label: 'TLIB',
url: 'https://www.brazzers.com/sites/view/id/51/teens-like-it-big',
description: "Whether they know it or not, teens love big stiff cocks in their tight pussies. Nothing goes better together than a tight, willing teen and a huge dick. In her bedroom or sneaking out to her boyfriend's, teens just want it all. Cum inside to see greedy sluts get more than they asked for",
network_id: 'brazzers',
},
{
id: 'zzseries',
name: 'ZZ Series',
label: 'ZZS',
url: 'https://www.brazzers.com/sites/view/id/81/zz-series',
description: 'This is the spot for all our high-end content. ZZ series is exclusive footage that offers only the best in terms of story, stars and action. Check out the hottest porn-stars having the nastiest sex here at the ZZ series',
network_id: 'brazzers',
},
{
id: 'mommygotboobs',
name: 'Mommy Got Boobs',
label: 'MGB',
url: 'https://www.brazzers.com/sites/view/id/10/mommy-got-boobs',
description: "When hubby's away MILFS will play. Older women just crave cock, and they're experienced enough to know that only a young stud will do. Big-titted sluts everywhere are sucking and fucking in secret, giving it away to anybody they can. At Mommy Got Boobs, you can get some MILF of your own.",
network_id: 'brazzers',
},
{
id: 'milfslikeitbig',
name: 'Milfs Like it Big',
label: 'MLIB',
url: 'https://www.brazzers.com/sites/view/id/36/milfs-like-it-big',
description: "When hubby's away milfy will play. These bored housewives want to get fucked and they want it now. They're experienced and know what they want. America's suburbs are full of these cum-de-sacs just waiting to get laid. Their round tits and thick asses are just begging for it. Cum inside, but don't park out front!",
network_id: 'brazzers',
},
{
id: 'bigtitsinuniform',
name: 'Big Tits In Uniform',
label: 'BTIU',
url: 'https://www.brazzers.com/sites/view/id/73/big-tits-in-uniform',
description: "Big titted wonders are all around us, doing the toughest jobs in the tightest uniforms. Look at them just bursting out of that blouse, or over there, bulging under that nurse's uniform. You know when those tight uniforms come off these sluts go wild, sucking and fucking cocks left and right, their big tits just bouncing. I can't wait to punch the clock.",
network_id: 'brazzers',
},
{
id: 'doctoradventures',
name: 'Doctor Adventures',
label: 'DA',
url: 'https://www.brazzers.com/sites/view/id/5/doctor-adventures',
description: 'Ever had fantasies about fucking your hot doctor? Live out your fantasies on doctoradventures.com. Countless doctor, patient scenarios come to life on this site with the sexiest and bustiest doctors imaginable! This is your one stop for the best in doctor porn in the world!',
network_id: 'brazzers',
},
{
id: 'brazzersexxtra',
name: 'Brazzers Exxtra',
label: 'BEX',
url: 'https://www.brazzers.com/sites/view/id/152/brazzers-exxtra',
description: "\"Brazzers Exxtra\" is a doorway to new, unseen hardcore content! There are countless Brazzers videos that were not released throughout the years and we haven't been able to show them to you until now. Random videos staring the world's most popular pornstars, fresh new industry faces and a whole lot more! We'll even throw in an occasional free video from our friends at Mofos, Twisty's and Babes! Check it all out and let us know what you think. If you want more, we'll get it for you!",
network_id: 'brazzers',
},
{
id: 'bigtitsinsports',
name: 'Big Tits In Sports',
label: 'BTIS',
url: 'https://www.brazzers.com/sites/view/id/54/big-tits-in-sports',
description: 'Watch them bounce, watch them score and look at the way they handle those balls! Big tits in sports is here and so are the best big titted, athletic babes. Facials on the court and threesomes on the field, these busty sluts are ready for anything, even if it means playing dirty. Could you take them 1 on 1?',
network_id: 'brazzers',
},
{
id: 'brazzersvault',
name: 'Brazzers Vault',
label: 'BZV',
url: 'https://www.brazzers.com/sites/view/id/56/brazzers-vault',
description: "We've got a whole super computer full of this stuff, technicians are working round the clock in the basement just to keep the thing from overheating. Yeah, this porno is hot. We need to get it out of before the whole thing melts down, that's why it's on the net, for you our loyal Brazzers Members. All the best scenes from all the best girls. In the World. Period.",
network_id: 'brazzers',
},
{
id: 'bigbuttslikeitbig',
name: 'Big Butts Like It Big',
label: 'BBLIB',
url: 'https://www.brazzers.com/sites/view/id/53/big-butts-like-it-big',
description: "You have to pair like with like. And big butts have to have big dicks to go with them. There's really no choice for these big round asses and the babes who fuck with them. Big assed bitches love it hard and deep, and won't have it any other way. Let the ass stuffing begin.",
network_id: 'brazzers',
},
{
id: 'bigwetbutts',
name: 'Big Wet Butts',
label: 'BWB',
url: 'https://www.brazzers.com/sites/view/id/8/big-wet-butts',
description: 'A nice, big, round butt is a special shape. Begging for powerful doggy style or straight anal penetration, cover a big butt in oil and it becomes a big wet butt, a true rarity. Watch these soft, tight asses get slathered and pounded like you only wish you could. Look at it bounce!',
network_id: 'brazzers',
},
{
id: 'daywithapornstar',
name: 'Day With A Pornstar',
label: 'DWP',
url: 'https://www.brazzers.com/sites/view/id/59/day-with-a-pornstar',
description: "We all know what our favorite stars can do on camera. We're familiar with the way they fuck and suck. What you don't get to see is what they do on their own time. Day With a Porn-star will show you everything, from crazy parties to total babe pals. Nobody else has access like this, it's the closest you get to living the dream.",
network_id: 'brazzers',
},
{
id: 'dirtymasseur',
name: 'Dirty Masseur',
label: 'DM',
url: 'https://www.brazzers.com/sites/view/id/150/dirty-masseur',
description: "Take a moment and unwind. Lay down, relax, and enjoy watching and wanking to these luscious Brazzers beauties getting good and greasy. Boobs, butts, and other lady-parts are at their prettiest when shimmering with slick oil. Book an appointment, and slide on in with a lubed babe. Believe me when I say, you'll have the happiest of endings...",
network_id: 'brazzers',
},
{
id: 'hotandmean',
name: 'Hot And Mean',
label: 'HAM',
url: 'https://www.brazzers.com/sites/view/id/78/hot-and-mean',
description: "The hottest bitches run together. Hot, mean lesbians love to fuck each other and hate each other for being so beautiful. These lesbo sluts can't get enough pussy and love girl on girl action. Forget the dicks, these chicks don't need 'em. You can watch though, they love that.",
network_id: 'brazzers',
},
{
id: 'brazzersenespanol',
name: 'Brazzers en Español',
label: 'BZES',
url: 'https://www.brazzers.com/sites/view/id/157/brazzers-en-espanol',
description: 'Brazzers en Español - El mejor sitio porno en alta definición del mundo ¡Ofreciéndole los vídeos para adultos en alta definición, descargables y en streaming, más exclusivos de Internet! Brazzers cuenta con las estrellas porno más sexys a través de los sitios más calientes en la red. Las estrellas porno y las escenas más calientes en internet. ¡Tendrá acceso a más sexo anal, tetas grandes y culos calientes de los que jamás soñó!',
network_id: 'brazzers',
},
{
id: 'brazzerslive',
name: 'Brazzers Live',
label: 'ZZL',
url: 'https://www.brazzers.com/sites/view/id/156/brazzers-live',
description: 'Brazzers is the industry leader for premium porn that breaks the mold. Pioneering its legendary LIVE SHOWS, ZZ is constantly redefining what hardcore erotica is about. Our wild fuck marathons are loaded with the steamiest improvised sex around. Catch a bevy of naked bodacious babes who ravage the biggest dicks with ease and in real-time. Our monster cock hunks rise to the occasion and feed these ravenous vixens who possess an insatiable appetite for cum.',
network_id: 'brazzers',
},
{
id: 'sexproadventures',
name: 'SexPro Adventures',
label: 'SPA',
url: 'https://www.brazzers.com/sites/view/id/23/sexpro-adventures',
description: "Having trouble with your dick-style? The sex pros are here and they'll teach you everything you need to know to be a better man. At your place or theirs, these sluts just want to have a good time. Don't worry, she's a professional.",
network_id: 'brazzers',
},
{
id: 'shesgonnasquirt',
name: 'Shes Gonna Squirt',
label: 'SGS',
url: 'https://www.brazzers.com/sites/view/id/151/shes-gonna-squirt',
description: "Enter the wet world of female ejaculation at shesgonnasquirt! Exclusive hardcore porn of your top pornstars squirting will excite you beyond belief. She's Gonna Squirt is home to the best in HD squirting sex videos. How to make a girl's pussy squirt is an art and should no longer remain a mystery, so join now to become a master.",
network_id: 'brazzers',
},
{
id: 'assesinpublic',
name: 'Asses In Public',
label: 'AIP',
url: 'https://www.brazzers.com/sites/view/id/50/asses-in-public',
description: "Sex in public can present its challenges, never fear, we're willing to accept them. There's something hot about asses out in the street that we just can't deny. Porn-stars fucking on public or just hot girls showing their asses in the airport, we've got both and then some. Asses in Public has the roundest asses and the biggest tits just hanging out, where WILL we show up next?",
network_id: 'brazzers',
},
{
id: 'bustyz',
name: 'Bustyz',
label: 'BTZ',
url: 'https://www.brazzers.com/sites/view/id/6/bustyz',
description: "If the internet was a town we'd have the biggest tits around. We still do though, because Bustyz features only the best endowed porn stars in solo and group action. Watch these big-titted babes take cock, suck twat and show off their massive jugs. Real or fake, we don't judge, everyone's welcome under the big tit tent",
network_id: 'brazzers',
},
{
id: 'bustyandreal',
name: 'Busty & Real',
label: 'BAR',
url: 'https://www.brazzers.com/sites/view/id/2/busty-real',
description: "Sometimes you need to take a break from the silicon football set. Busty and real has all the real jugs you need. Round. Soft. and as real as they come. These babes are rocking exactly what momma gave them. They've not afraid to show off their assets and get slammed with dick in the process.",
network_id: 'brazzers',
},
{
id: 'hotchicksbigasses',
name: 'Hot Chicks Big Asses',
label: 'HCBA',
url: 'https://www.brazzers.com/sites/view/id/7/hot-chicks-big-asses',
description: 'Everyone gather round; the giant ass. A babe can be hot in a lot of ways and having a big round ass is one of the best. All shapes, sizes and types these girls are the best of the best. Round, supple, jiggling asses taking on dicks and other pussies in equal measure.',
network_id: 'brazzers',
},
{
id: 'cfnm',
name: 'CFNM',
label: 'CFNM',
url: 'https://www.brazzers.com/sites/view/id/154/cfnm',
description: "Welcome to the world of clothed female sluts fucking, humiliating and dominating naked men, giving them a dose of what it feels like to be owned. If you love women with power dominating wimpy guys and showing them who's boss; women who crave for cock but get it exactly how they want it, that's what you'll find here. Simply put, the guys don't fuck the women, the women fuck the guys and make them feel like whores!",
network_id: 'brazzers',
},
{
id: 'jugfuckers',
name: 'JugFuckers',
label: 'JF',
url: 'https://www.brazzers.com/sites/view/id/12/jugfuckers',
description: "Like a sex hot-dog, a big dick fits nicely between two soft, round tits. Tit-fucking isn't easy and never will be. Our girls are pros and take big loads on their faces and tits with a smile. From DD to the smallest things going, we've got every type of tit- fuck around.",
network_id: 'brazzers',
},
{
id: 'teenslikeitblack',
name: 'Teens Like It Black',
label: 'TLB',
url: 'https://www.brazzers.com/sites/view/id/57/teens-like-it-black',
description: "Teens just wanna piss their parents off; no rules, spring break, big black cocks. They love pushing things to the limit, how big and black can it be? Only teen girls know. Watch them get more than they bargained for, long black cocks drilling their tight, inexperienced pussies. It's an epic fuck when the biggest and the tightest meet.",
network_id: 'brazzers',
},
{
id: 'racksandblacks',
name: 'Racks & Blacks',
label: 'RAB',
url: 'https://www.brazzers.com/sites/view/id/11/racks-blacks',
description: "All the interracial action you need is here. Big 'ol black cocks ramming and jamming pussies to the limit. All types of different girls fall prey to the venerable black dick. Wet pussies and fat asses? Bring it on. There's nothing our stable of asses can't handle, they'll keep cumming and cumming.",
network_id: 'brazzers',
},
{
id: 'buttsandblacks',
name: 'Butts & Blacks',
label: 'BAB',
url: 'https://www.brazzers.com/sites/view/id/3/butts-blacks',
description: "Giant black dicks paired with round asses and garnished with the tightest pussies of all colors. Butts and Blacks delivers on its name sake, only the biggest dicks rocking the thickest chicks. These round honeys can take it all in and bounce around like it's a pogo stick. Come check out these soft round asses getting the attention they deserve.",
network_id: 'brazzers',
},
// JULES JORDAN
{
id: 'julesjordan',
@ -14,15 +279,6 @@ exports.seed = knex => Promise.resolve()
network_id: 'julesjordan',
},
// KINK
{
// for scene URLs only
id: 'kink',
name: 'Kink',
label: 'kink',
url: 'https://www.kink.com',
description: 'Authentic Bondage & Real BDSM Porn Videos. Demystifying and celebrating alternative sexuality by providing the most authentic kinky videos. Experience the other side of porn.',
network_id: 'kink',
},
{
id: 'thirtyminutesoftorment',
name: '30 Minutes of Torment',
@ -327,15 +583,6 @@ exports.seed = knex => Promise.resolve()
parameters: JSON.stringify({ tourId: 9 }),
},
// PRIVATE
{
// for scene URLs only
id: 'private',
name: 'Private',
label: 'Privat',
url: 'https://www.private.com',
description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
network_id: 'private',
},
{
id: 'analintroductions',
name: 'Anal Introductions',

56
seeds/tags.js
View File

@ -20,6 +20,10 @@ exports.seed = knex => Promise.resolve()
tag: 'anal fisting',
alias_for: null,
},
{
tag: 'anal prolapse',
alias_for: null,
},
{
tag: 'anal toy',
alias_for: null,
@ -32,6 +36,10 @@ exports.seed = knex => Promise.resolve()
tag: 'ass licking',
alias_for: null,
},
{
tag: 'athletic',
alias_for: null,
},
{
tag: 'ATM',
alias_for: null,
@ -144,6 +152,10 @@ exports.seed = knex => Promise.resolve()
tag: 'electric shock',
alias_for: null,
},
{
tag: 'enhanced boobs',
alias_for: null,
},
{
tag: 'European',
alias_for: null,
@ -273,11 +285,11 @@ exports.seed = knex => Promise.resolve()
alias_for: null,
},
{
tag: 'pussy eating',
tag: 'piercings',
alias_for: null,
},
{
tag: 'anal prolapse',
tag: 'pussy eating',
alias_for: null,
},
{
@ -376,6 +388,10 @@ exports.seed = knex => Promise.resolve()
tag: 'TP',
alias_for: null,
},
{
tag: 'trimmed',
alias_for: null,
},
{
tag: 'uniform',
alias_for: null,
@ -392,6 +408,10 @@ exports.seed = knex => Promise.resolve()
tag: 'white',
alias_for: null,
},
{
tag: 'wife',
alias_for: null,
},
]))
.then(() => knex('tags').insert([
// ALIASES
@ -467,6 +487,14 @@ exports.seed = knex => Promise.resolve()
tag: 'blowjob (double)',
alias_for: 'double blowjob',
},
{
tag: 'boob job',
alias_for: 'enhanced boobs',
},
{
tag: 'boobjob',
alias_for: 'enhanced boobs',
},
{
tag: 'brunettes',
alias_for: 'brunette',
@ -591,10 +619,22 @@ exports.seed = knex => Promise.resolve()
tag: 'drool',
alias_for: 'saliva',
},
{
tag: 'enhanced',
alias_for: 'enhanced boobs',
},
{
tag: 'facials',
alias_for: 'facial',
},
{
tag: 'fake boobs',
alias_for: 'enhanced boobs',
},
{
tag: 'fake tits',
alias_for: 'enhanced boobs',
},
{
tag: 'flogging',
alias_for: 'corporal punishment',
@ -643,6 +683,14 @@ exports.seed = knex => Promise.resolve()
tag: 'MFF',
alias_for: 'FMF',
},
{
tag: 'piercing',
alias_for: 'piercings',
},
{
tag: 'pierced',
alias_for: 'piercings',
},
{
tag: 'prolapse',
alias_for: 'anal prolapse',
@ -755,6 +803,10 @@ exports.seed = knex => Promise.resolve()
tag: 'toys',
alias_for: 'toy',
},
{
tag: 'trimmed pussy',
alias_for: 'trimmed',
},
{
tag: 'triple penetration',
alias_for: 'TP',

6
src/fetch-scene.js
View File

@ -9,14 +9,16 @@ const scrapers = require('./scrapers');
async function findSite(url) {
const { origin } = new URL(url);
const site = await knex('sites').where({ url: origin }).first();
const site = await knex('sites').where({ url: origin }).first()
// scenes might be listed on network site, let network scraper find channel site
|| await knex('networks').where({ url: origin }).first();
return {
id: site.id,
name: site.name,
description: site.description,
url: site.url,
networkId: site.network_id,
networkId: site.network_id || site.id,
};
}

118
src/scrapers/brazzers.js Normal file
View File

@ -0,0 +1,118 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
function scrape(html, site, upcoming) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.release-card.scene').toArray();
return sceneElements.reduce((acc, element) => {
const isUpcoming = $(element).find('.icon-upcoming.active').length === 1;
if ((upcoming && !isUpcoming) || (!upcoming && isUpcoming)) {
return acc;
}
const sceneLinkElement = $(element).find('a');
const url = `https://www.brazzers.com${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title');
const shootId = url.split('/').slice(-3, -2)[0];
const date = moment.utc($(element).find('time').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.model-names a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($(element).find('.label-rating .like-amount').text());
const dislikes = Number($(element).find('.label-rating .dislike-amount').text());
return acc.concat({
url,
shootId,
title,
actors,
date,
rating: {
likes,
dislikes,
},
site,
});
}, []);
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const shootId = url.split('/').slice(-3, -2)[0];
const title = $('.scene-title[itemprop="name"]').text();
const description = $('#scene-description p[itemprop="description"]')
.contents()
.first()
.text()
.trim();
const date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate();
const actors = $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60;
const likes = Number($('.label-rating .like').text());
const dislikes = Number($('.label-rating .dislike').text());
const siteElement = $('.niche-site-logo');
const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`;
const siteName = siteElement.attr('title');
const rawTags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [tags, channelSite] = await Promise.all([
matchTags(rawTags),
knex('sites')
.where({ url: siteUrl })
.orWhere({ name: siteName })
.first(),
]);
return {
url,
shootId,
title,
actors,
date,
rating: {
likes,
dislikes,
},
site: channelSite || site,
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/`);
return scrape(res.body.toString(), site, false);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/`);
return scrape(res.body.toString(), site, true);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchUpcoming,
fetchScene,
};

2
src/scrapers/index.js
View File

@ -1,5 +1,6 @@
'use strict';
const brazzers = require('./brazzers');
const julesjordan = require('./julesjordan');
const kink = require('./kink');
const legalporno = require('./legalporno');
@ -9,6 +10,7 @@ const vixen = require('./vixen');
const xempire = require('./xempire');
module.exports = {
brazzers,
julesjordan,
kink,
legalporno,

13
src/scrapers/private.js
View File

@ -57,13 +57,16 @@ async function scrapeScene(html, url, site) {
const siteElement = $('.content-wrapper .logos-sites a');
const siteUrl = siteElement.attr('href').slice(0, -1);
const siteName = siteElement.text();
const channelSite = await knex('sites')
.where({ url: siteUrl })
.orWhere({ name: siteName })
.first();
const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = await matchTags(rawTags);
const [tags, channelSite] = await Promise.all([
matchTags(rawTags),
knex('sites')
.where({ url: siteUrl })
.orWhere({ name: siteName })
.first(),
]);
return {
url,

4
src/scrapers/boilerplate.js → src/scrapers/template.js
View File

@ -9,7 +9,7 @@ const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.scenes-latest');
const sceneElements = $('.scenes-latest').toArray();
return sceneElements.map((element) => {
return {
@ -30,7 +30,7 @@ function scrapeLatest(html, site) {
function scrapeUpcoming(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.scenes-upcoming');
const sceneElements = $('.scenes-upcoming').toArray();
return sceneElements.map((element) => {
return {