DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added Private scraper. Added Vixen scraper to repository.

This commit is contained in:
ThePendulum 2019-04-04 04:00:28 +02:00
parent 439d3225ec
commit b3beeef3e4
14 changed files with 545 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/default.js
View File

@ -18,6 +18,8 @@ module.exports = {
]], ]],
'legalporno', 'legalporno',
'pervcity', 'pervcity',
'private',
'vixen',
'xempire', 'xempire',
], ],
columns: [ columns: [

12
migrations/20190325001339_releases.js
View File

@ -9,6 +9,14 @@ exports.up = knex => Promise.resolve()
.references('id') .references('id')
.inTable('actors'); .inTable('actors');
})) }))
.then(() => knex.schema.createTable('directors', (table) => {
table.increments('id', 8);
table.string('name');
table.integer('alias_for', 8)
.references('id')
.inTable('directors');
}))
.then(() => knex.schema.createTable('tags', (table) => { .then(() => knex.schema.createTable('tags', (table) => {
table.string('tag', 20) table.string('tag', 20)
.primary(); .primary();
@ -57,6 +65,10 @@ exports.up = knex => Promise.resolve()
table.date('date'); table.date('date');
table.text('description'); table.text('description');
table.integer('director', 8)
.references('id')
.inTable('directors');
table.integer('duration') table.integer('duration')
.unsigned(); .unsigned();

14
seeds/networks.js
View File

@ -28,10 +28,10 @@ exports.seed = knex => Promise.resolve()
description: '', description: '',
}, },
{ {
id: 'xempire', id: 'private',
name: 'XEmpire', name: 'Private',
url: 'https://www.xempire.com', url: 'https://www.private.com',
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!', description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
}, },
{ {
id: 'vixen', id: 'vixen',
@ -39,4 +39,10 @@ exports.seed = knex => Promise.resolve()
url: 'https://www.vixen.com/', url: 'https://www.vixen.com/',
description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.', description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
}, },
{
id: 'xempire',
name: 'XEmpire',
url: 'https://www.xempire.com',
description: 'XEmpire.com brings you today\'s top pornstars in beautifully shot, HD sex scenes across 4 unique porn sites of gonzo porn, interracial, lesbian & erotica!',
},
])); ]));

245
seeds/sites.js
View File

@ -15,6 +15,7 @@ exports.seed = knex => Promise.resolve()
}, },
// KINK // KINK
{ {
// for scene URLs only
id: 'kink', id: 'kink',
name: 'Kink', name: 'Kink',
label: 'kink', label: 'kink',
@ -25,7 +26,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'thirtyminutesoftorment', id: 'thirtyminutesoftorment',
name: '30 Minutes of Torment', name: '30 Minutes of Torment',
label: '30mtor', label: '30MTor',
url: 'https://www.kink.com/channel/30minutesoftorment', url: 'https://www.kink.com/channel/30minutesoftorment',
description: 'Thick-Muscled Men Endure 30 Minutes Of BDSM Torment By A Pain-Inducing Dom. Can they take 30 Minutes of Torment? Watch as top gay pornstars take on the challenge of a lifetime. Bondage, BDSM, punishment, huge insertions, & more!', description: 'Thick-Muscled Men Endure 30 Minutes Of BDSM Torment By A Pain-Inducing Dom. Can they take 30 Minutes of Torment? Watch as top gay pornstars take on the challenge of a lifetime. Bondage, BDSM, punishment, huge insertions, & more!',
network_id: 'kink', network_id: 'kink',
@ -33,7 +34,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'boundgangbangs', id: 'boundgangbangs',
name: 'Bound Gangbangs', name: 'Bound Gangbangs',
label: 'boundg', label: 'BoundG',
url: 'https://www.kink.com/channel/boundgangbangs', url: 'https://www.kink.com/channel/boundgangbangs',
description: 'Poweless whores tied in bondage and stuffed with a cock in every hole. At BoundGangbangs women get surprise extreme gangbangs, blindfolds, deepthroat blowjobs, sex punishment, bondage, double penetration and interracial sex.', description: 'Poweless whores tied in bondage and stuffed with a cock in every hole. At BoundGangbangs women get surprise extreme gangbangs, blindfolds, deepthroat blowjobs, sex punishment, bondage, double penetration and interracial sex.',
network_id: 'kink', network_id: 'kink',
@ -41,7 +42,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'boundgods', id: 'boundgods',
name: 'Bound Gods', name: 'Bound Gods',
label: 'bngods', label: 'BnGods',
url: 'https://www.kink.com/channel/boundgods', url: 'https://www.kink.com/channel/boundgods',
description: 'Muscle Studs Are Bound, Gagged & Spread For A Deep Cock Pounding. Not even the most rock hard muscled studs can escape punishment & submission on BoundGods.com Watch the hottest studs get tied down, fucked & submitted.', description: 'Muscle Studs Are Bound, Gagged & Spread For A Deep Cock Pounding. Not even the most rock hard muscled studs can escape punishment & submission on BoundGods.com Watch the hottest studs get tied down, fucked & submitted.',
network_id: 'kink', network_id: 'kink',
@ -49,7 +50,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'boundinpublic', id: 'boundinpublic',
name: 'Bound in Public', name: 'Bound in Public',
label: 'boundp', label: 'BoundP',
url: 'https://www.kink.com/channel/boundinpublic', url: 'https://www.kink.com/channel/boundinpublic',
description: 'Cum Starved Sluts Humiliated And Fucked Hard In Public By Hung Studs.', description: 'Cum Starved Sluts Humiliated And Fucked Hard In Public By Hung Studs.',
network_id: 'kink', network_id: 'kink',
@ -57,7 +58,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'brutalsessions', id: 'brutalsessions',
name: 'Brutal Sessions', name: 'Brutal Sessions',
label: 'brutal', label: 'Brutal',
url: 'https://www.kink.com/channel/brutalsessions', url: 'https://www.kink.com/channel/brutalsessions',
description: "Hardcore BDSM jam packed with XXX fucking in bondage! We're taking dungeon sex beyond the castle!", description: "Hardcore BDSM jam packed with XXX fucking in bondage! We're taking dungeon sex beyond the castle!",
network_id: 'kink', network_id: 'kink',
@ -65,7 +66,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'buttmachineboys', id: 'buttmachineboys',
name: 'Butt Machine Boys', name: 'Butt Machine Boys',
label: 'bmboys', label: 'BMBoys',
url: 'https://www.kink.com/channel/buttmachineboys', url: 'https://www.kink.com/channel/buttmachineboys',
description: 'Powerful Fucking Machines Pound Hot Men Hard & Deep.', description: 'Powerful Fucking Machines Pound Hot Men Hard & Deep.',
network_id: 'kink', network_id: 'kink',
@ -73,7 +74,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'devicebondage', id: 'devicebondage',
name: 'Device Bondage', name: 'Device Bondage',
label: 'dvbond', label: 'DvBond',
url: 'https://www.kink.com/channel/devicebondage', url: 'https://www.kink.com/channel/devicebondage',
description: 'The Domination Of Sluts In Barbaric Metal Devices. Device Bondage takes BDSM porn to new levels with extreme restraints & unique devices with beautiful pornstars to huge, forced squirting orgasms.', description: 'The Domination Of Sluts In Barbaric Metal Devices. Device Bondage takes BDSM porn to new levels with extreme restraints & unique devices with beautiful pornstars to huge, forced squirting orgasms.',
network_id: 'kink', network_id: 'kink',
@ -81,7 +82,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'devinebitches', id: 'devinebitches',
name: 'Devine Bitches', name: 'Devine Bitches',
label: 'dbitch', label: 'DBitch',
url: 'https://www.kink.com/channel/divinebitches', url: 'https://www.kink.com/channel/divinebitches',
description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.', description: 'Beautiful Women Dominate Submissive Men With Pain, Humiliation And Strap-On Fucking. The best in femdom and bondage. Men on Divine Bitches respond with obedience, ass worship, cunt worship, oral servitude, pantyhose worship, and foot worship.',
network_id: 'kink', network_id: 'kink',
@ -89,7 +90,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'electrosluts', id: 'electrosluts',
name: 'Electrosluts', name: 'Electrosluts',
label: 'esluts', label: 'ESluts',
url: 'https://www.kink.com/channel/electrosluts', url: 'https://www.kink.com/channel/electrosluts',
description: 'Lezdoms Take Submissive Sluts To Their Limits, Shocking & Tormenting Their Wet Hot Pussies. Pornstars live out their electric bondage fantasies while dominatrixes use electrodes, paddles, caddle prods, & more to bring them to intense orgasms!', description: 'Lezdoms Take Submissive Sluts To Their Limits, Shocking & Tormenting Their Wet Hot Pussies. Pornstars live out their electric bondage fantasies while dominatrixes use electrodes, paddles, caddle prods, & more to bring them to intense orgasms!',
network_id: 'kink', network_id: 'kink',
@ -97,7 +98,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'everythingbutt', id: 'everythingbutt',
name: 'Everything Butt', name: 'Everything Butt',
label: 'evbutt', label: 'EvButt',
url: 'https://www.kink.com/channel/everythingbutt', url: 'https://www.kink.com/channel/everythingbutt',
description: 'Gaping Anal Holes Are Stuffed & Stretched To The Max. Anal Fisting, Enemas & Rimming Has Never Tasted So Good. EverythingButt.com explores the extreme limits of FemDom lesbian anal. Watch asses get destroyed by brutal fistings, huge insertions, double anal & more!', description: 'Gaping Anal Holes Are Stuffed & Stretched To The Max. Anal Fisting, Enemas & Rimming Has Never Tasted So Good. EverythingButt.com explores the extreme limits of FemDom lesbian anal. Watch asses get destroyed by brutal fistings, huge insertions, double anal & more!',
network_id: 'kink', network_id: 'kink',
@ -105,7 +106,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'familiestied', id: 'familiestied',
name: 'Families Tied', name: 'Families Tied',
label: 'famtie', label: 'FamTie',
url: 'https://www.kink.com/channel/familiestied', url: 'https://www.kink.com/channel/familiestied',
description: 'Intense BDSM family role play threesomes & more.', description: 'Intense BDSM family role play threesomes & more.',
network_id: 'kink', network_id: 'kink',
@ -113,7 +114,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'footworship', id: 'footworship',
name: 'Foot Worship', name: 'Foot Worship',
label: 'footws', label: 'FootWS',
url: 'https://www.kink.com/channel/footworship', url: 'https://www.kink.com/channel/footworship',
description: 'Satisfy Your Foot Fetish With The Kinkiest Foot Action. Enjoy Trampling, Foot Jobs, High Heels, And Pantyhose.', description: 'Satisfy Your Foot Fetish With The Kinkiest Foot Action. Enjoy Trampling, Foot Jobs, High Heels, And Pantyhose.',
network_id: 'kink', network_id: 'kink',
@ -121,7 +122,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'fuckedandbound', id: 'fuckedandbound',
name: 'Fucked and Bound', name: 'Fucked and Bound',
label: 'fbound', label: 'FBound',
url: 'https://www.kink.com/channel/fuckedandbound', url: 'https://www.kink.com/channel/fuckedandbound',
description: 'Extreme Anal, Rope Bondage, & Brutal Face Fucking.', description: 'Extreme Anal, Rope Bondage, & Brutal Face Fucking.',
network_id: 'kink', network_id: 'kink',
@ -129,7 +130,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'fuckingmachines', id: 'fuckingmachines',
name: 'Fucking Machines', name: 'Fucking Machines',
label: 'fmachi', label: 'FMachi',
url: 'https://www.kink.com/channel/fuckingmachines', url: 'https://www.kink.com/channel/fuckingmachines',
description: 'Machines Fucking Squirting Pussies With Extreme Insertions. Fucking Machines is the ultimate hardcore sex toy porn. Huge dildos strapped to sex machines relentlessly fucking pornstars to real squirting oragsms!', description: 'Machines Fucking Squirting Pussies With Extreme Insertions. Fucking Machines is the ultimate hardcore sex toy porn. Huge dildos strapped to sex machines relentlessly fucking pornstars to real squirting oragsms!',
network_id: 'kink', network_id: 'kink',
@ -137,7 +138,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'hardcoregangbang', id: 'hardcoregangbang',
name: 'Hardcore Gangbang', name: 'Hardcore Gangbang',
label: 'hardgb', label: 'HardGB',
url: 'https://www.kink.com/channel/hardcoregangbang', url: 'https://www.kink.com/channel/hardcoregangbang',
description: "Where all women's hardcore gangbang fantasies come true. Watch extreme, brutal gangbangs with pornstars, models, & MILFs that crave cock in every hole. HardcoreGangbang.com has the best creampie gang bangs online.", description: "Where all women's hardcore gangbang fantasies come true. Watch extreme, brutal gangbangs with pornstars, models, & MILFs that crave cock in every hole. HardcoreGangbang.com has the best creampie gang bangs online.",
network_id: 'kink', network_id: 'kink',
@ -145,7 +146,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'hogtied', id: 'hogtied',
name: 'Hogtied', name: 'Hogtied',
label: 'hogtie', label: 'Hogtie',
url: 'https://www.kink.com/channel/hogtied', url: 'https://www.kink.com/channel/hogtied',
description: 'Your favorite girls restrained with rope, punished & trained. Hogtied is the original extreme bondage porn website. Watch top pornstars and pain sluts in brutal bondage, getting tormented, and forced to orgasm!', description: 'Your favorite girls restrained with rope, punished & trained. Hogtied is the original extreme bondage porn website. Watch top pornstars and pain sluts in brutal bondage, getting tormented, and forced to orgasm!',
network_id: 'kink', network_id: 'kink',
@ -153,7 +154,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'kinkuniversity', id: 'kinkuniversity',
name: 'Kink University', name: 'Kink University',
label: 'kinkun', label: 'KinkUn',
url: 'https://www.kink.com/channel/kinkuniversity', url: 'https://www.kink.com/channel/kinkuniversity',
description: 'Learn BDSM Technical Skills & Theories From Respected Teachers In The Kink Community. Learn BDSM skills and improve your sex techniques. Video tutorials feature top sex ed experts and hardcore demos on topics from bondage to relationships.', description: 'Learn BDSM Technical Skills & Theories From Respected Teachers In The Kink Community. Learn BDSM skills and improve your sex techniques. Video tutorials feature top sex ed experts and hardcore demos on topics from bondage to relationships.',
network_id: 'kink', network_id: 'kink',
@ -161,7 +162,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'meninpain', id: 'meninpain',
name: 'Men In Pain', name: 'Men In Pain',
label: 'meninp', label: 'MenInP',
url: 'https://www.kink.com/channel/meninpain', url: 'https://www.kink.com/channel/meninpain',
description: 'Submissive Men Violated With Verbal Humiliation And Harsh Punishment By Beautiful Dominatrices.', description: 'Submissive Men Violated With Verbal Humiliation And Harsh Punishment By Beautiful Dominatrices.',
network_id: 'kink', network_id: 'kink',
@ -169,7 +170,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'menonedge', id: 'menonedge',
name: 'Men on Edge', name: 'Men on Edge',
label: 'moedge', label: 'MOEdge',
url: 'https://www.kink.com/channel/menonedge', url: 'https://www.kink.com/channel/menonedge',
description: "Hot Guys Begging To Cum Are Brought To The Edge Of Complete Submission And Allowed To Blow Their Loads. Men on Edge has perfected the art of gay BDSM & edging porn. Watch straight men bound up & edged by dominant gay pornstars until they can't help but cum!", description: "Hot Guys Begging To Cum Are Brought To The Edge Of Complete Submission And Allowed To Blow Their Loads. Men on Edge has perfected the art of gay BDSM & edging porn. Watch straight men bound up & edged by dominant gay pornstars until they can't help but cum!",
network_id: 'kink', network_id: 'kink',
@ -177,7 +178,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'nakedkombat', id: 'nakedkombat',
name: 'Naked Kombat', name: 'Naked Kombat',
label: 'kombat', label: 'Kombat',
url: 'https://www.kink.com/channel/nakedkombat', url: 'https://www.kink.com/channel/nakedkombat',
description: 'Fight Fit Studs Go Head To Head In A Battle For Dominance. The Loser Gets Pinned And Punish Fucked Without Mercy', description: 'Fight Fit Studs Go Head To Head In A Battle For Dominance. The Loser Gets Pinned And Punish Fucked Without Mercy',
network_id: 'kink', network_id: 'kink',
@ -185,7 +186,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'publicdisgrace', id: 'publicdisgrace',
name: 'Public Disgrace', name: 'Public Disgrace',
label: 'pubdis', label: 'PubDis',
url: 'https://www.kink.com/channel/publicdisgrace', url: 'https://www.kink.com/channel/publicdisgrace',
description: 'Women Bound Stripped And Punished In Public Get Hardcore Fucked Where Everyone Can See. Unscripted public humiliation & punishment of submissive slaves in real life locations. PublicDisgrace features the best outdoor BDSM & voyeur porn!', description: 'Women Bound Stripped And Punished In Public Get Hardcore Fucked Where Everyone Can See. Unscripted public humiliation & punishment of submissive slaves in real life locations. PublicDisgrace features the best outdoor BDSM & voyeur porn!',
network_id: 'kink', network_id: 'kink',
@ -193,7 +194,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'sadisticrope', id: 'sadisticrope',
name: 'Sadistic Rope', name: 'Sadistic Rope',
label: 'sadisr', label: 'SadisR',
url: 'https://www.kink.com/channel/sadisticrope', url: 'https://www.kink.com/channel/sadisticrope',
description: 'Innocence Taken By Extreme Rope Bondage, Hardcore BDSM And Pussy-Destroying Orgasms.', description: 'Innocence Taken By Extreme Rope Bondage, Hardcore BDSM And Pussy-Destroying Orgasms.',
network_id: 'kink', network_id: 'kink',
@ -201,7 +202,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'sexandsubmission', id: 'sexandsubmission',
name: 'Sex and Submission', name: 'Sex and Submission',
label: 'sexsub', label: 'SexSub',
url: 'https://www.kink.com/channel/sexandsubmission', url: 'https://www.kink.com/channel/sexandsubmission',
description: 'Submissive Sluts Are Dominated With Rough Sex And Bondage. Real pornstars, hardcore bondage, master & slave roles are what SexAndSubmission.com is all about. Watch submissive sluts give in to total domination!', description: 'Submissive Sluts Are Dominated With Rough Sex And Bondage. Real pornstars, hardcore bondage, master & slave roles are what SexAndSubmission.com is all about. Watch submissive sluts give in to total domination!',
network_id: 'kink', network_id: 'kink',
@ -209,7 +210,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'thetrainingofo', id: 'thetrainingofo',
name: 'The Training of O', name: 'The Training of O',
label: 'traino', label: 'TrainO',
url: 'https://www.kink.com/channel/thetrainingofo', url: 'https://www.kink.com/channel/thetrainingofo',
description: 'Slaves Are Trained And Rewarded With Hardcore Bondage And Sex. Watch real pornstars undergo extreme slave training through hardcore bondage & BDSM porn. The Training of O is the ultimate slave / master experience!', description: 'Slaves Are Trained And Rewarded With Hardcore Bondage And Sex. Watch real pornstars undergo extreme slave training through hardcore bondage & BDSM porn. The Training of O is the ultimate slave / master experience!',
network_id: 'kink', network_id: 'kink',
@ -217,7 +218,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'theupperfloor', id: 'theupperfloor',
name: 'The Upper Floor', name: 'The Upper Floor',
label: 'upperf', label: 'UFloor',
url: 'https://www.kink.com/channel/theupperfloor', url: 'https://www.kink.com/channel/theupperfloor',
description: 'Trained slaves serve the house and their master in intense BDSM and kinky threesomes. The Upper Floor is a voyeuristic look into BDSM and fetish porn shoots with real submissive pornstars living out their kinky fantasies live on cam.', description: 'Trained slaves serve the house and their master in intense BDSM and kinky threesomes. The Upper Floor is a voyeuristic look into BDSM and fetish porn shoots with real submissive pornstars living out their kinky fantasies live on cam.',
network_id: 'kink', network_id: 'kink',
@ -225,7 +226,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'tspussyhunts', id: 'tspussyhunts',
name: 'TS Pussy Hunters', name: 'TS Pussy Hunters',
label: 'tspuss', label: 'TSPuss',
url: 'https://www.kink.com/channel/tspussyhunters', url: 'https://www.kink.com/channel/tspussyhunters',
description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.', description: 'Hot TS cocks prey on the wet pussies of submissive ladies who are fucked hard till they cum. Dominant TS femme fatales with the hardest dicks, the softest tits, and the worst intentions dominate, bind, and punish bitches on the ultimate transfucking porn site.',
network_id: 'kink', network_id: 'kink',
@ -233,7 +234,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'tsseduction', id: 'tsseduction',
name: 'TS Seduction', name: 'TS Seduction',
label: 'tseduc', label: 'TSeduc',
url: 'https://www.kink.com/channel/tsseduction', url: 'https://www.kink.com/channel/tsseduction',
description: 'Sexy TS Women With Huge Cocks Dominate The Holes Of Straight Boys. Real TS women who are drop-dead gorgeous from their pretty faces to their big tits to their hard TS cocks. TS Seduction is the ultimate in transsexual bondage porn.', description: 'Sexy TS Women With Huge Cocks Dominate The Holes Of Straight Boys. Real TS women who are drop-dead gorgeous from their pretty faces to their big tits to their hard TS cocks. TS Seduction is the ultimate in transsexual bondage porn.',
network_id: 'kink', network_id: 'kink',
@ -241,7 +242,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'ultimatesurrender', id: 'ultimatesurrender',
name: 'Ultimate Surrender', name: 'Ultimate Surrender',
label: 'ultsur', label: 'UltSur',
url: 'https://www.kink.com/channel/ultimatesurrender', url: 'https://www.kink.com/channel/ultimatesurrender',
description: 'Competitive Female Wrestling Where The Loser Gets Strap-On Punish Fucked. Ultimate Surrender features hardcore naked female wrestling porn videos where the winner gets to dominate the loser with some kinky lesbian FemDom!', description: 'Competitive Female Wrestling Where The Loser Gets Strap-On Punish Fucked. Ultimate Surrender features hardcore naked female wrestling porn videos where the winner gets to dominate the loser with some kinky lesbian FemDom!',
network_id: 'kink', network_id: 'kink',
@ -249,7 +250,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'waterbondage', id: 'waterbondage',
name: 'Water Bondage', name: 'Water Bondage',
label: 'waterb', label: 'WaterB',
url: 'https://www.kink.com/channel/waterbondage', url: 'https://www.kink.com/channel/waterbondage',
description: 'Helpless Bound Beauties Sprayed, Dunked And Tormented Until They Cum Hard & Wet.', description: 'Helpless Bound Beauties Sprayed, Dunked And Tormented Until They Cum Hard & Wet.',
network_id: 'kink', network_id: 'kink',
@ -257,7 +258,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'whippedass', id: 'whippedass',
name: 'Whipped Ass', name: 'Whipped Ass',
label: 'whippd', label: 'WhipdA',
url: 'https://www.kink.com/channel/whippedass', url: 'https://www.kink.com/channel/whippedass',
description: 'Beautiful Submissive Sluts Take A Hard Fucking From Powerful Dominant Women. Watch brutal lesbian dominatrixes push submissive sluts to their orgasmic breaking points on WhippedAss! Hardcore fisting, huge strapons & face sitting!', description: 'Beautiful Submissive Sluts Take A Hard Fucking From Powerful Dominant Women. Watch brutal lesbian dominatrixes push submissive sluts to their orgasmic breaking points on WhippedAss! Hardcore fisting, huge strapons & face sitting!',
network_id: 'kink', network_id: 'kink',
@ -265,7 +266,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'wiredpussy', id: 'wiredpussy',
name: 'Wired Pussy', name: 'Wired Pussy',
label: 'wiredp', label: 'WiredP',
url: 'https://www.kink.com/channel/wiredpussy', url: 'https://www.kink.com/channel/wiredpussy',
description: 'Gorgeous Women Submit To Electricity, Are Zapped, Shocked & Prodded To Orgasm.', description: 'Gorgeous Women Submit To Electricity, Are Zapped, Shocked & Prodded To Orgasm.',
network_id: 'kink', network_id: 'kink',
@ -274,7 +275,7 @@ exports.seed = knex => Promise.resolve()
{ {
id: 'legalporno', id: 'legalporno',
name: 'LegalPorno', name: 'LegalPorno',
label: 'legalp', label: 'LegalP',
url: 'https://www.legalporno.com', url: 'https://www.legalporno.com',
description: 'The Best HD Porn For You!', description: 'The Best HD Porn For You!',
network_id: 'legalporno', network_id: 'legalporno',
@ -325,6 +326,145 @@ exports.seed = knex => Promise.resolve()
network_id: 'pervcity', network_id: 'pervcity',
parameters: JSON.stringify({ tourId: 9 }), parameters: JSON.stringify({ tourId: 9 }),
}, },
// PRIVATE
{
// for scene URLs only
id: 'private',
name: 'Private',
label: 'Privat',
url: 'https://www.private.com',
description: 'Private is the best source for adult movies and videos. Featuring the most popular hardcore adult stars in hundreds of porn movies, Private.com delivers...',
network_id: 'private',
},
{
id: 'analintroductions',
name: 'Anal Introductions',
label: 'AIntro',
description: 'Private\'s Anal Introductions is all about ass. Watch these girls get their asses broken in by fat hard cocks! Hot double penetrations, gaping wide assholes and anal creampies are all standard in this exclusive site. Many of these girls have never had cock in their ass before, while others are real addicts and can only cum when being savagely sodomised. Watch which girls can take it... Private style.',
url: 'https://www.private.com/site/anal-introductions',
network_id: 'private',
},
{
id: 'iconfessfiless',
name: 'I Confess Files',
label: 'confes',
description: 'From the heart of the UK comes found footage exclusively provided to private.com which will shock and offend some viewers. Reality, perversion and unnatural lust come together perhaps as never before.',
url: 'https://www.private.com/site/i-confess-files',
network_id: 'private',
},
{
id: 'missionasspossible',
name: 'Mission: Ass Possible',
label: 'misass',
description: 'From the streets of Europe, Private\'s team of professionals find and exploit clueless young sluts, for some great sex and for your viewing pleasure. See what young hot chicks will do when their desire for easy fame or money makes them vulnerable to some of the craziest schemes and plots imaginable. Private\'s hung studs are on a mission for ass and hungry for fun. All part of the Private network of sites.',
url: 'https://www.private.com/site/mission-ass-possible',
network_id: 'private',
},
{
id: 'russianfakeagent',
name: 'Russian Fake Agent',
label: 'rsfake',
description: 'Direct from Russia, young naïve women pursue their dream of visas, Hollywood and fame. Eager to please and willing to do anything to get their break. Unfortunately, it’s a case of lies, sex and videotape as these gullible hotties perform for us. If these girls only knew the truth!',
url: 'https://www.private.com/site/russian-fake-agent',
network_id: 'private',
},
{
id: 'sexonthebeach',
name: 'Sex on the Beach',
label: 'sexbch',
description: 'Amazing locations and steamy sex in the sun, www.privatetropics.com is a celebration of tropical lust and sand covered sluts. From Private\'s exclusive line of scenes in exotic countries, watch what happens when our hot models go naked and native.',
url: 'https://www.private.com/site/sex-on-the-beach',
network_id: 'private',
},
{
id: 'tightandteen',
name: 'Tight and Teen',
label: 'tightt',
description: 'Europe\'s number one teen offering and part of the Private network of sites, Tight and Teen takes you to the place that every father dreads, as 18+ teens discover just how much fun pleasing a hung stud can be. Fresh tight pussies, virgin anal initiations and teen face fuckings all brought to you exclusively by Europe\'s leading adult brand.',
url: 'https://www.private.com/site/tight-and-teen',
network_id: 'private',
},
{
id: 'blacksonsluts',
name: 'Blacks on Sluts',
label: 'blslut',
description: 'See what happens when European women discover hung black studs looking to breed. Blacks on Sluts is 100% white slut and cheating wives versus huge black dicks giving them something they will never forget. Private puts its stamp on these women as our stallions stretch them to their limits.',
url: 'https://www.private.com/site/blacks-on-sluts',
network_id: 'private',
},
{
id: 'privatefetish',
name: 'Private Fetish',
label: 'privft',
description: 'www.privatefetish.com is here to give you that taste of dark desire that you secretly crave. Domination and Submission, Pleasure and Pain, are the drivers in this hardcore dungeon. What turns you on most? Being forced to beg for release from a sexy dominatrix or making that bitch next door beg for your cock? All part of Private\'s network of sites.',
url: 'https://www.private.com/site/private-fetish',
network_id: 'private',
},
{
id: 'privatemilfs',
name: 'Private MILFs',
label: 'prmilf',
description: 'Part of the awesome network of Private sites, Private MILFs is all about moms who are getting what they need when their limp dicked husbands can\'t perform. From their daughters\' stud boyfriends to their husbands\' hung black co-workers to their sons\' friends, no one is safe from their cravings for hard cock and salty cum.',
url: 'https://www.private.com/site/private-milfs',
network_id: 'private',
},
{
id: 'russianteenass',
name: 'Russian Teen Ass',
label: 'rusass',
description: 'Many people say that Russian girls are the most beautiful in the world, and at www.russianteenass.com we show you why. These sexy Soviet newcomers are ready to work hard for their visa and their big shot at stardom. These barely 18+ girls know what they want and Private gives them an exclusive opportunity to get it. From Russia with lust, come see these girls in action!',
url: 'https://www.private.com/site/russian-teen-ass',
network_id: 'private',
},
{
id: 'privatestars',
name: 'Private Stars',
label: 'prstar',
description: 'Welcome to Private Stars. The name speaks for itself as only top-model babes and perfect girls can join this select group of stars, all shot in HD. Part of the Private network of sites, Private Stars brings you a sneak peek into the life of seductive glamour girls who could have easily stepped from the catwalks of Paris or Milan and straight into a world of torrid sex.',
url: 'https://www.private.com/site/private-stars',
network_id: 'private',
},
// VIXEN
{
id: 'vixen',
name: 'Vixen',
label: 'Vixen',
description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
url: 'https://www.vixen.com',
network_id: 'vixen',
},
{
id: 'blacked',
name: 'Blacked',
label: 'Blackd',
description: 'Porn videos of beautiful girls in first time interracial porn videos. BLACKED has the hottest pornstars in HD sex videos.',
url: 'https://www.blacked.com',
network_id: 'vixen',
},
{
id: 'tushy',
name: 'Tushy',
label: 'Tushy',
description: 'Watch the world\'s best HD Anal videos! Featuring beautiful, never before seen girls in first time anal. Exclusively on Tushy.com',
url: 'https://www.tushy.com',
network_id: 'vixen',
},
{
id: 'blackedraw',
name: 'Blacked Raw',
label: 'BlkRaw',
description: 'Experience real women in interracial sex videos. Passionate sex with beautiful pornstars. No photoshop just the highest quality porn. Everything you see is real.',
url: 'https://www.blackedraw.com',
network_id: 'vixen',
},
{
id: 'tushyraw',
name: 'Tushy Raw',
label: 'TshRaw',
description: 'Anal sex videos with beautiful models and pornstars being fucked in the ass. TUSHY RAW features famous pornstars in high quality anal porn videos.',
url: 'https://www.tushyraw.com',
network_id: 'vixen',
},
// XEMPIRE // XEMPIRE
{ {
id: 'hardx', id: 'hardx',
@ -358,45 +498,4 @@ exports.seed = knex => Promise.resolve()
url: 'https://www.lesbianx.com', url: 'https://www.lesbianx.com',
network_id: 'xempire', network_id: 'xempire',
}, },
// VIXEN
{
id: 'vixen',
name: 'Vixen',
label: 'vixen',
description: 'Vixen.com features the world’s finest cinematic adult films with 4K quality and high-end erotic photography.',
url: 'https://www.vixen.com',
network_id: 'vixen',
},
{
id: 'blacked',
name: 'Blacked',
label: 'blackd',
description: 'Porn videos of beautiful girls in first time interracial porn videos. BLACKED has the hottest pornstars in HD sex videos.',
url: 'https://www.blacked.com',
network_id: 'vixen',
},
{
id: 'tushy',
name: 'Tushy',
label: 'tushy',
description: 'Watch the world\'s best HD Anal videos! Featuring beautiful, never before seen girls in first time anal. Exclusively on Tushy.com',
url: 'https://www.tushy.com',
network_id: 'vixen',
},
{
id: 'blackedraw',
name: 'Blacked Raw',
label: 'blkraw',
description: 'Experience real women in interracial sex videos. Passionate sex with beautiful pornstars. No photoshop just the highest quality porn. Everything you see is real.',
url: 'https://www.blackedraw.com',
network_id: 'vixen',
},
{
id: 'tushyraw',
name: 'Tushy Raw',
label: 'tshraw',
description: 'Anal sex videos with beautiful models and pornstars being fucked in the ass. TUSHY RAW features famous pornstars in high quality anal porn videos.',
url: 'https://www.tushyraw.com',
network_id: 'vixen',
},
])); ]));

60
seeds/tags.js
View File

@ -124,6 +124,10 @@ exports.seed = knex => Promise.resolve()
tag: 'ebony', tag: 'ebony',
alias_for: null, alias_for: null,
}, },
{
tag: 'European',
alias_for: null,
},
{ {
tag: 'facefucking', tag: 'facefucking',
alias_for: null, alias_for: null,
@ -136,6 +140,10 @@ exports.seed = knex => Promise.resolve()
tag: 'facials', tag: 'facials',
alias_for: 'facial', alias_for: 'facial',
}, },
{
tag: 'FMF',
alias_for: null,
},
{ {
tag: 'gangbang', tag: 'gangbang',
alias_for: null, alias_for: null,
@ -180,6 +188,10 @@ exports.seed = knex => Promise.resolve()
tag: 'MILF', tag: 'MILF',
alias_for: null, alias_for: null,
}, },
{
tag: 'MFM',
alias_for: null,
},
{ {
tag: 'missionary', tag: 'missionary',
alias_for: null, alias_for: null,
@ -208,6 +220,10 @@ exports.seed = knex => Promise.resolve()
tag: 'rough', tag: 'rough',
alias_for: null, alias_for: null,
}, },
{
tag: 'Russian',
alias_for: null,
},
{ {
tag: 'schoolgirl', tag: 'schoolgirl',
alias_for: null, alias_for: null,
@ -260,12 +276,24 @@ exports.seed = knex => Promise.resolve()
tag: 'TP', tag: 'TP',
alias_for: null, alias_for: null,
}, },
{
tag: 'uniform',
alias_for: null,
},
{
tag: 'voyeur',
alias_for: null,
},
{ {
tag: 'white', tag: 'white',
alias_for: null, alias_for: null,
}, },
])) ]))
.then(() => knex('tags').insert([ .then(() => knex('tags').insert([
{
tag: '2-on-1',
alias_for: 'MFM',
},
{ {
tag: '3+ on 1', tag: '3+ on 1',
alias_for: 'gangbang', alias_for: 'gangbang',
@ -282,6 +310,10 @@ exports.seed = knex => Promise.resolve()
tag: 'ass to mouth', tag: 'ass to mouth',
alias_for: 'ATM', alias_for: 'ATM',
}, },
{
tag: 'BGB',
alias_for: 'MFM',
},
{ {
tag: 'big ass', tag: 'big ass',
alias_for: 'big butt', alias_for: 'big butt',
@ -414,10 +446,22 @@ exports.seed = knex => Promise.resolve()
tag: 'gapes (gaping asshole)', tag: 'gapes (gaping asshole)',
alias_for: 'gaping', alias_for: 'gaping',
}, },
{
tag: 'huge tits',
alias_for: 'big boobs',
},
{ {
tag: 'huge toys', tag: 'huge toys',
alias_for: 'toys', alias_for: 'toys',
}, },
{
tag: 'MMF',
alias_for: 'MFM',
},
{
tag: 'MFF',
alias_for: 'FMF',
},
{ {
tag: 'red head', tag: 'red head',
alias_for: 'redhead', alias_for: 'redhead',
@ -442,10 +486,18 @@ exports.seed = knex => Promise.resolve()
tag: 'school girl', tag: 'school girl',
alias_for: 'schoolgirl', alias_for: 'schoolgirl',
}, },
{
tag: 'shaved pussy',
alias_for: 'shaved',
},
{ {
tag: 'small tits', tag: 'small tits',
alias_for: 'small boobs', alias_for: 'small boobs',
}, },
{
tag: 'spitroast',
alias_for: 'MFM',
},
{ {
tag: 'standing doggystyle', tag: 'standing doggystyle',
alias_for: 'standing doggy style', alias_for: 'standing doggy style',
@ -462,6 +514,14 @@ exports.seed = knex => Promise.resolve()
tag: 'teens', tag: 'teens',
alias_for: 'teen', alias_for: 'teen',
}, },
{
tag: 'tiny boobs',
alias_for: 'small boobs',
},
{
tag: 'tiny tits',
alias_for: 'small boobs',
},
{ {
tag: 'toys', tag: 'toys',
alias_for: 'toy', alias_for: 'toy',

10
src/argv.js
View File

@ -4,6 +4,16 @@ const yargs = require('yargs');
const { argv } = yargs const { argv } = yargs
.command('npm start') .command('npm start')
.option('networks', {
describe: 'Networks to include (overrides config)',
type: 'array',
alias: 'network',
})
.option('sites', {
describe: 'Sites to include (overrides config)',
type: 'array',
alias: 'site',
})
.option('render', { .option('render', {
describe: 'Fetch data without rendering interface', describe: 'Fetch data without rendering interface',
type: 'boolean', type: 'boolean',

26
src/fetch-releases.js
View File

@ -3,6 +3,7 @@
const config = require('config'); const config = require('config');
const moment = require('moment'); const moment = require('moment');
const argv = require('./argv');
const knex = require('./knex'); const knex = require('./knex');
const scrapers = require('./scrapers'); const scrapers = require('./scrapers');
@ -38,6 +39,14 @@ function curateSites(sites) {
} }
async function accumulateIncludedSites() { async function accumulateIncludedSites() {
if (argv.networks || argv.sites) {
const rawSites = await knex('sites')
.whereIn('id', argv.sites || [])
.orWhereIn('network_id', argv.networks || []);
return curateSites(rawSites);
}
const included = destructConfigNetworks(config.include); const included = destructConfigNetworks(config.include);
const rawSites = await knex('sites') const rawSites = await knex('sites')
@ -47,9 +56,11 @@ async function accumulateIncludedSites() {
return curateSites(rawSites); return curateSites(rawSites);
} }
async function getExistingReleases() { async function getStoredReleases(siteId, limit) {
return knex('releases'); return knex('releases')
// .where('date', '>', new Date(2019, 2, 26)); .where({ site_id: siteId })
.orderBy('date', 'desc')
.limit(limit);
} }
async function storeReleases(releases) { async function storeReleases(releases) {
@ -60,6 +71,7 @@ async function storeReleases(releases) {
title: release.title, title: release.title,
date: release.date, date: release.date,
description: release.description, description: release.description,
director: release.director,
duration: release.duration, duration: release.duration,
likes: release.rating && release.rating.likes, likes: release.rating && release.rating.likes,
dislikes: release.rating && release.rating.dislikes, dislikes: release.rating && release.rating.dislikes,
@ -80,20 +92,22 @@ async function storeReleases(releases) {
async function fetchReleases() { async function fetchReleases() {
const sites = await accumulateIncludedSites(); const sites = await accumulateIncludedSites();
const releases = await getExistingReleases(); // const releases = await getExistingReleases();
const scenesPerSite = await Promise.all(sites.map(async (site) => { const scenesPerSite = await Promise.all(sites.map(async (site) => {
const scraper = scrapers[site.id] || scrapers[site.networkId]; const scraper = scrapers[site.id] || scrapers[site.networkId];
if (scraper) { if (scraper) {
const storedReleases = await getStoredReleases(site.id, 100);
const [latest, upcoming] = await Promise.all([ const [latest, upcoming] = await Promise.all([
scraper.fetchLatest(site), scraper.fetchLatest(site, storedReleases),
scraper.fetchUpcoming ? scraper.fetchUpcoming(site) : [], scraper.fetchUpcoming ? scraper.fetchUpcoming(site) : [],
]); ]);
console.log(`${latest.length} published releases and ${upcoming.length} upcoming releases found`); console.log(`${latest.length} published releases and ${upcoming.length} upcoming releases found`);
await storeReleases(latest, releases); await storeReleases(latest);
return [...latest, ...upcoming]; return [...latest, ...upcoming];
} }

49
src/scrapers/boilerplate.js
View File

@ -9,14 +9,63 @@ const { matchTags } = require('../tags');
function scrapeLatest(html, site) { function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.scenes-latest');
return sceneElements.map((element) => {
return {
url,
shootId,
title,
actors,
date,
rating: {
likes,
dislikes,
stars,
},
site,
};
});
} }
function scrapeUpcoming(html, site) { function scrapeUpcoming(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.scenes-upcoming');
return sceneElements.map((element) => {
return {
url,
shootId,
title,
actors,
date,
rating: {
likes,
dislikes,
stars,
},
site,
};
});
} }
function scrapeScene(html, url, site) { function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true }); const $ = cheerio.load(html, { normalizeWhitespace: true });
return {
url,
shootId,
title,
actors,
director: '',
date,
rating: {
likes,
dislikes,
stars,
},
site,
};
} }
async function fetchLatest(site) { async function fetchLatest(site) {

6
src/scrapers/index.js
View File

@ -1,17 +1,19 @@
'use strict'; 'use strict';
const xempire = require('./xempire');
const julesjordan = require('./julesjordan'); const julesjordan = require('./julesjordan');
const kink = require('./kink'); const kink = require('./kink');
const legalporno = require('./legalporno'); const legalporno = require('./legalporno');
const pervcity = require('./pervcity'); const pervcity = require('./pervcity');
const privateNetwork = require('./private'); // reserved keyword
const vixen = require('./vixen'); const vixen = require('./vixen');
const xempire = require('./xempire');
module.exports = { module.exports = {
xempire,
julesjordan, julesjordan,
kink, kink,
legalporno, legalporno,
pervcity, pervcity,
private: privateNetwork,
vixen, vixen,
xempire,
}; };

6
src/scrapers/kink.js
View File

@ -65,8 +65,10 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const channelSite = await knex('sites').where({ id: sitename }).first(); const [channelSite, tags] = await Promise.all([
const tags = await matchTags(rawTags); knex('sites').where({ id: sitename }).first(),
matchTags(rawTags),
]);
return { return {
url, url,

100
src/scrapers/private.js Normal file
View File

@ -0,0 +1,100 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.content-wrapper.container .scene').toArray();
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('a[data-track="TITLE_LINK"]');
const url = sceneLinkElement.attr('href');
const title = sceneLinkElement.text();
const shootId = url.split('/').slice(-1)[0];
const date = moment.utc($(element).find('.scene-date'), 'MM/DD/YYYY').toDate();
const actors = $(element).find('a[data-track="PORNSTAR_LINK"]').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const likes = Number($(element).find('.scene-votes').text());
return {
url,
shootId,
title,
actors,
date,
rating: {
likes,
dislikes: 0,
},
site,
};
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const shootId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate()
const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const description = $('.video-wrapper meta[itemprop="description"]').attr('content');
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds);
const likes = Number($('.content-desc #social-actions #likes').text());
const siteElement = $('.content-wrapper .logos-sites a');
const siteUrl = siteElement.attr('href').slice(0, -1);
const siteName = siteElement.text();
const channelSite = await knex('sites')
.where({ url: siteUrl })
.orWhere({ name: siteName })
.first();
const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = await matchTags(rawTags);
return {
url,
shootId,
title,
date,
actors,
description,
duration,
tags,
rating: {
likes,
dislikes: 0,
},
site: channelSite || site,
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/`);
return scrapeLatest(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchScene,
};

90
src/scrapers/vixen.js Normal file
View File

@ -0,0 +1,90 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const stateObject = $('script:contains("INITIAL_STATE")');
const { videos: scenes } = JSON.parse(stateObject.html().trim().slice(27, -1));
return scenes.map((scene) => {
const shootId = scene.newId;
const title = scene.title;
const url = `${site.url}${scene.targetUrl}`;
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
const actors = scene.models;
const stars = Number(scene.textRating) / 2;
return {
url,
shootId,
title,
actors,
date,
rating: {
stars,
},
site,
};
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const { pathname, search } = new URL(url);
const stateObject = $('script:contains("INITIAL_STATE")');
const data = JSON.parse(stateObject.html().trim().slice(27, -1));
const scene = data.page.data[`${pathname}${search}`].data.video;
const shootId = scene.newId;
const title = scene.title;
const date = new Date(scene.releaseDate);
const actors = scene.models;
const stars = scene.totalRateVal;
const rawTags = scene.tags;
const tags = await matchTags(rawTags);
const duration = scene.runLength;
const director = scene.directorNames;
return {
url,
shootId,
title,
actors,
director,
date,
duration,
tags,
rating: {
stars,
},
site,
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/videos`);
return scrapeLatest(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchScene,
};

12
src/scrapers/xempire.js
View File

@ -34,6 +34,7 @@ function scrape(html, site) {
shootId, shootId,
title, title,
actors, actors,
director: 'Mason',
date, date,
rating: { rating: {
likes, likes,
@ -62,6 +63,7 @@ async function scrapeSceneFallback($, url, site) {
title, title,
date, date,
actors, actors,
director: 'Mason',
description, description,
tags, tags,
rating: { rating: {
@ -108,6 +110,7 @@ async function scrapeScene(html, url, site) {
title, title,
date, date,
actors, actors,
director: 'Mason',
description, description,
duration, duration,
tags, tags,
@ -118,10 +121,15 @@ async function scrapeScene(html, url, site) {
}; };
} }
async function fetchLatest(site) { async function fetchLatest(site, storedReleases) {
const res = await bhttp.get(`${site.url}/en/videos`); const res = await bhttp.get(`${site.url}/en/videos`);
const releases = scrape(res.body.toString(), site);
return scrape(res.body.toString(), site); const storedShootIds = new Set(storedReleases.map(release => release.shoot_id));
const newReleases = releases.filter(release => !storedShootIds.has(release.shootId));
console.log(newReleases);
} }
async function fetchUpcoming(site) { async function fetchUpcoming(site) {

3
src/tags.js
View File

@ -5,7 +5,8 @@ const knex = require('./knex');
async function matchTags(rawTags) { async function matchTags(rawTags) {
const tagEntries = await knex('tags') const tagEntries = await knex('tags')
.select(knex.raw('ifnull(original.tag, tags.tag) as tag'), knex.raw('ifnull(original.tag, tags.tag) as tag')) .select(knex.raw('ifnull(original.tag, tags.tag) as tag'), knex.raw('ifnull(original.tag, tags.tag) as tag'))
.whereIn('tags.tag', rawTags.map(tag => tag.toLowerCase())) .whereIn('tags.tag', rawTags)
.orWhereIn('tags.tag', rawTags.map(tag => tag.toLowerCase()))
.leftJoin('tags as original', 'tags.alias_for', 'original.tag'); .leftJoin('tags as original', 'tags.alias_for', 'original.tag');
return tagEntries.map(({ tag }) => tag); return tagEntries.map(({ tag }) => tag);