Added Naughty America scraper.
This commit is contained in:
parent
aa33c83b83
commit
763ffb341a
59
README.md
59
README.md
|
|
@ -113,6 +113,65 @@ The latest releases from your favorite porn studios in one place.
|
|||
* Share My BF
|
||||
* She's A Freak
|
||||
* Stranded Teens
|
||||
* **Naughty America**
|
||||
* 2 Chicks Same Time
|
||||
* American Daydreams
|
||||
* Anal College
|
||||
* Asian 1 On 1
|
||||
* Ass Masterpiece
|
||||
* Big Cock Bully
|
||||
* Diary of a Milf
|
||||
* Diary of a Nanny
|
||||
* Dirty Wives Club
|
||||
* Fast Times
|
||||
* Housewife 1 on 1
|
||||
* I Have a Wife
|
||||
* LA Sluts
|
||||
* Latin Adultery
|
||||
* Latina Step Mom
|
||||
* Lesbian Girl on Girl
|
||||
* Live Gym Cam
|
||||
* Live Naughty Milf
|
||||
* Live Naughty Nurse
|
||||
* Live Naughty Secretary
|
||||
* Live Naughty Student
|
||||
* Live Naughty Teacher
|
||||
* Live Party Girl
|
||||
* Milf Sugar Babes Classic
|
||||
* My Dad's Hot Girlfriend
|
||||
* My Daughter's Hot Friend
|
||||
* My First Sex Teacher
|
||||
* My Friend's Hot Girl
|
||||
* My Friend's Hot Mom
|
||||
* My Girl Loves Anal
|
||||
* My Girlfriend's Busty Friend
|
||||
* My Naughty Latin Maid
|
||||
* My Naughty Massage
|
||||
* My Sister's Hot Friend
|
||||
* My Wife Is My Pornstar
|
||||
* My Wife's Hot Friend
|
||||
* Naughty America
|
||||
* Naughty Athletics
|
||||
* Naughty Bookworms
|
||||
* Naughty Country Girls
|
||||
* Naughty Flipside
|
||||
* Naughty Office
|
||||
* Naughty Rich Girls
|
||||
* Naughty Weddings
|
||||
* Neighbor Affair
|
||||
* Open Family
|
||||
* Perfect Fucking Strangers Classic
|
||||
* Seduced By A Cougar
|
||||
* Sleazy Stepdad
|
||||
* Slut Step Mom
|
||||
* Slut Step Sister
|
||||
* Socal Coeds
|
||||
* Teens Love Cream
|
||||
* The Passenger
|
||||
* Tonight's Girlfriend
|
||||
* Watch Your Mom
|
||||
* Watch Your Wife
|
||||
* Wives on Vacation
|
||||
* **Perv City**
|
||||
* Anal Overdose
|
||||
* Banging Beauties
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ module.exports = {
|
|||
]],
|
||||
'legalporno',
|
||||
'mofos',
|
||||
'naughtyamerica',
|
||||
'pervcity',
|
||||
['private', [
|
||||
'analintroductions',
|
||||
|
|
@ -35,6 +36,7 @@ module.exports = {
|
|||
'russianteenass',
|
||||
'privatestars',
|
||||
]],
|
||||
'realitykings',
|
||||
'vixen',
|
||||
'xempire',
|
||||
],
|
||||
|
|
|
|||
|
|
@ -51,6 +51,12 @@ exports.seed = knex => Promise.resolve()
|
|||
url: 'https://www.mofos.com',
|
||||
description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
|
||||
},
|
||||
{
|
||||
id: 'naughtyamerica',
|
||||
name: 'Naughty America',
|
||||
url: 'https://www.naughtyamerica.com',
|
||||
description: 'The best porn movies daily at Naughty America! Experience the most seductive porn stars in stunning virtual reality, 4K and HD porn videos!',
|
||||
},
|
||||
{
|
||||
id: 'pervcity',
|
||||
name: 'Perv City',
|
||||
|
|
|
|||
349
seeds/sites.js
349
seeds/sites.js
|
|
@ -839,6 +839,355 @@ exports.seed = knex => Promise.resolve()
|
|||
description: 'Watch intimate video diaries from hot chicks who like nothing better than to show and tell, whether it be their bangin’ new boobs or their sweet little pussies. These are real life confessions, from real life sluts, who want you to know all their dirtiest secrets.',
|
||||
network_id: 'mofos',
|
||||
},
|
||||
// NAUGHTY AMERICA
|
||||
{
|
||||
id: 'myfriendshotmom',
|
||||
name: "My Friend's Hot Mom",
|
||||
url: 'https://www.naughtyamerica.com/site/my-friend-s-hot-mom',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'slutstepmom',
|
||||
name: 'Slut Step Mom',
|
||||
url: 'https://www.naughtyamerica.com/site/slut-step-mom',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'openfamily',
|
||||
name: 'Open Family',
|
||||
url: 'https://www.naughtyamerica.com/site/open-family',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'sleazystepdad',
|
||||
name: 'Sleazy Stepdad',
|
||||
url: 'https://www.naughtyamerica.com/site/sleazy-stepdad',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'watchyourmom',
|
||||
name: 'Watch Your Mom',
|
||||
url: 'https://www.naughtyamerica.com/site/watch-your-mom',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'bigcockbully',
|
||||
name: 'Big Cock Bully',
|
||||
url: 'https://www.naughtyamerica.com/site/big-cock-bully',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mysistershotfriend',
|
||||
name: "My Sister's Hot Friend",
|
||||
url: 'https://www.naughtyamerica.com/site/my-sister-s-hot-friend',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'myfirstsexteacher',
|
||||
name: 'My First Sex Teacher',
|
||||
url: 'https://www.naughtyamerica.com/site/my-first-sex-teacher',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'slutstepsister',
|
||||
name: 'Slut Step Sister',
|
||||
url: 'https://www.naughtyamerica.com/site/slut-step-sister',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'teenslovecream',
|
||||
name: 'Teens Love Cream',
|
||||
url: 'https://www.naughtyamerica.com/site/teens-love-cream',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'latinastepmom',
|
||||
name: 'Latina Step Mom',
|
||||
url: 'https://www.naughtyamerica.com/site/latina-step-mom',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'seducedbyacougar',
|
||||
name: 'Seduced By A Cougar',
|
||||
url: 'https://www.naughtyamerica.com/site/seduced-by-a-cougar',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mydaughtershotfriend',
|
||||
name: "My Daughter's Hot Friend",
|
||||
url: 'https://www.naughtyamerica.com/site/my-daughter-s-hot-friend',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'lasluts',
|
||||
name: 'LA Sluts',
|
||||
url: 'https://www.naughtyamerica.com/site/la-sluts',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mywifeismypornstar',
|
||||
name: 'My Wife Is My Pornstar',
|
||||
url: 'https://www.naughtyamerica.com/site/my-wife-is-my-pornstar',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'watchyourwife',
|
||||
name: 'Watch Your Wife',
|
||||
url: 'https://www.naughtyamerica.com/site/watch-your-wife',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'tonightsgirlfriendclassic',
|
||||
name: "Tonight's Girlfriend",
|
||||
url: 'https://www.naughtyamerica.com/site/tonight-s-girlfriend-classic',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'wivesonvacation',
|
||||
name: 'Wives on Vacation',
|
||||
url: 'https://www.naughtyamerica.com/site/wives-on-vacation',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyweddings',
|
||||
name: 'Naughty Weddings',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-weddings',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'dirtywivesclub',
|
||||
name: 'Dirty Wives Club',
|
||||
url: 'https://www.naughtyamerica.com/site/dirty-wives-club',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mydadshotgirlfriend',
|
||||
name: "My Dad's Hot Girlfriend",
|
||||
url: 'https://www.naughtyamerica.com/site/my-dad-s-hot-girlfriend',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mygirllovesanal',
|
||||
name: 'My Girl Loves Anal',
|
||||
url: 'https://www.naughtyamerica.com/site/my-girl-loves-anal',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'analcollege',
|
||||
name: 'Anal College',
|
||||
url: 'https://www.naughtyamerica.com/site/anal-college',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'lesbiangirlongirl',
|
||||
name: 'Lesbian Girl on Girl',
|
||||
url: 'https://www.naughtyamerica.com/site/lesbian-girl-on-girl',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyoffice',
|
||||
name: 'Naughty Office',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-office',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'ihaveawife',
|
||||
name: 'I Have a Wife',
|
||||
url: 'https://www.naughtyamerica.com/site/i-have-a-wife',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtybookworms',
|
||||
name: 'Naughty Bookworms',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-bookworms',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'housewife1on1',
|
||||
name: 'Housewife 1 on 1',
|
||||
url: 'https://www.naughtyamerica.com/site/housewife-1-on-1',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mywifeshotfriend',
|
||||
name: "My Wife's Hot Friend",
|
||||
url: 'https://www.naughtyamerica.com/site/my-wife-s-hot-friend',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'latinadultery',
|
||||
name: 'Latin Adultery',
|
||||
url: 'https://www.naughtyamerica.com/site/latin-adultery',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'assmasterpiece',
|
||||
name: 'Ass Masterpiece',
|
||||
url: 'https://www.naughtyamerica.com/site/ass-masterpiece',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: '2chickssametime',
|
||||
name: '2 Chicks Same Time',
|
||||
url: 'https://www.naughtyamerica.com/site/2-chicks-same-time',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'myfriendshotgirl',
|
||||
name: "My Friend's Hot Girl",
|
||||
url: 'https://www.naughtyamerica.com/site/my-friend-s-hot-girl',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'neighboraffair',
|
||||
name: 'Neighbor Affair',
|
||||
url: 'https://www.naughtyamerica.com/site/neighbor-affair',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mygirlfriendsbustyfriend',
|
||||
name: "My Girlfriend's Busty Friend",
|
||||
url: 'https://www.naughtyamerica.com/site/my-girlfriend-s-busty-friend',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyathletics',
|
||||
name: 'Naughty Athletics',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-athletics',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mynaughtymassage',
|
||||
name: 'My Naughty Massage',
|
||||
url: 'https://www.naughtyamerica.com/site/my-naughty-massage',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'fasttimes',
|
||||
name: 'Fast Times',
|
||||
url: 'https://www.naughtyamerica.com/site/fast-times',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'thepassenger',
|
||||
name: 'The Passenger',
|
||||
url: 'https://www.naughtyamerica.com/site/the-passenger',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'milfsugarbabesclassic',
|
||||
name: 'Milf Sugar Babes Classic',
|
||||
url: 'https://www.naughtyamerica.com/site/milf-sugar-babes-classic',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'perfectfuckingstrangersclassic',
|
||||
name: 'Perfect Fucking Strangers Classic',
|
||||
url: 'https://www.naughtyamerica.com/site/perfect-fucking-strangers-classic',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'asian1on1',
|
||||
name: 'Asian 1 On 1',
|
||||
url: 'https://www.naughtyamerica.com/site/asian-1-on-1',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'americandaydreams',
|
||||
name: 'American Daydreams',
|
||||
url: 'https://www.naughtyamerica.com/site/american-daydreams',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'socalcoeds',
|
||||
name: 'Socal Coeds',
|
||||
url: 'https://www.naughtyamerica.com/site/socal-coeds',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtycountrygirls',
|
||||
name: 'Naughty Country Girls',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-country-girls',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'diaryofamilf',
|
||||
name: 'Diary of a Milf',
|
||||
url: 'https://www.naughtyamerica.com/site/diary-of-a-milf',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyrichgirls',
|
||||
name: 'Naughty Rich Girls',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-rich-girls',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'mynaughtylatinmaid',
|
||||
name: 'My Naughty Latin Maid',
|
||||
url: 'https://www.naughtyamerica.com/site/my-naughty-latin-maid',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyamerica',
|
||||
name: 'Naughty America',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-america',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'diaryofananny',
|
||||
name: 'Diary of a Nanny',
|
||||
url: 'https://www.naughtyamerica.com/site/diary-of-a-nanny',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'naughtyflipside',
|
||||
name: 'Naughty Flipside',
|
||||
url: 'https://www.naughtyamerica.com/site/naughty-flipside',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livepartygirl',
|
||||
name: 'Live Party Girl',
|
||||
url: 'https://www.naughtyamerica.com/site/live-party-girl',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livenaughtystudent',
|
||||
name: 'Live Naughty Student',
|
||||
url: 'https://www.naughtyamerica.com/site/live-naughty-student',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livenaughtysecretary',
|
||||
name: 'Live Naughty Secretary',
|
||||
url: 'https://www.naughtyamerica.com/site/live-naughty-secretary',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livegymcam',
|
||||
name: 'Live Gym Cam',
|
||||
url: 'https://www.naughtyamerica.com/site/live-gym-cam',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livenaughtyteacher',
|
||||
name: 'Live Naughty Teacher',
|
||||
url: 'https://www.naughtyamerica.com/site/live-naughty-teacher',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livenaughtymilf',
|
||||
name: 'Live Naughty Milf',
|
||||
url: 'https://www.naughtyamerica.com/site/live-naughty-milf',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
{
|
||||
id: 'livenaughtynurse',
|
||||
name: 'Live Naughty Nurse',
|
||||
url: 'https://www.naughtyamerica.com/site/live-naughty-nurse',
|
||||
network_id: 'naughtyamerica',
|
||||
},
|
||||
// PERVCITY
|
||||
{
|
||||
id: 'analoverdose',
|
||||
|
|
|
|||
|
|
@ -203,6 +203,14 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'cum licking',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'cum on butt',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'cum on boobs',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'cumshot',
|
||||
alias_for: null,
|
||||
|
|
@ -426,6 +434,10 @@ exports.seed = knex => Promise.resolve()
|
|||
alias_for: null,
|
||||
group_id: 'location',
|
||||
},
|
||||
{
|
||||
tag: 'outie pussy',
|
||||
alias_for: null,
|
||||
},
|
||||
{
|
||||
tag: 'pain',
|
||||
alias_for: null,
|
||||
|
|
@ -694,6 +706,10 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'blondes',
|
||||
alias_for: 'blonde hair',
|
||||
},
|
||||
{
|
||||
tag: 'blow job',
|
||||
alias_for: 'blowjob',
|
||||
},
|
||||
{
|
||||
tag: 'blowjobs',
|
||||
alias_for: 'blowjob',
|
||||
|
|
@ -766,6 +782,14 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'cum in mouth',
|
||||
alias_for: 'oral creampie',
|
||||
},
|
||||
{
|
||||
tag: 'cum on ass',
|
||||
alias_for: 'cum on butt',
|
||||
},
|
||||
{
|
||||
tag: 'cum on tits',
|
||||
alias_for: 'cum on boobs',
|
||||
},
|
||||
{
|
||||
tag: 'cum swallowing',
|
||||
alias_for: 'swallowing',
|
||||
|
|
@ -786,6 +810,10 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'deep throat',
|
||||
alias_for: 'deepthroat',
|
||||
},
|
||||
{
|
||||
tag: 'deepthroating',
|
||||
alias_for: 'deepthroat',
|
||||
},
|
||||
{
|
||||
tag: 'dildo',
|
||||
alias_for: 'toys',
|
||||
|
|
@ -982,6 +1010,10 @@ exports.seed = knex => Promise.resolve()
|
|||
tag: 'oral',
|
||||
alias_for: 'blowjob',
|
||||
},
|
||||
{
|
||||
tag: 'outie',
|
||||
alias_for: 'outie pussy',
|
||||
},
|
||||
{
|
||||
tag: 'piercing',
|
||||
alias_for: 'piercings',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
'use strict';
|
||||
|
||||
const twentyonesextury = require('./21sextury');
|
||||
const blowpass = require('./blowpass');
|
||||
const brazzers = require('./brazzers');
|
||||
const ddfnetwork = require('./ddfnetwork');
|
||||
|
|
@ -9,8 +10,8 @@ const legalporno = require('./legalporno');
|
|||
const mofos = require('./mofos');
|
||||
const pervcity = require('./pervcity');
|
||||
const privateNetwork = require('./private'); // reserved keyword
|
||||
const naughtyamerica = require('./naughtyamerica');
|
||||
const realitykings = require('./realitykings');
|
||||
const twentyonesextury = require('./21sextury');
|
||||
const vixen = require('./vixen');
|
||||
const xempire = require('./xempire');
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ module.exports = {
|
|||
mofos,
|
||||
pervcity,
|
||||
private: privateNetwork,
|
||||
naughtyamerica,
|
||||
realitykings,
|
||||
vixen,
|
||||
xempire,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
'use strict';
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const knex = require('../knex');
|
||||
const { matchTags } = require('../tags');
|
||||
|
||||
function titleExtractor(pathname) {
|
||||
const components = pathname.split('/')[2].split('-');
|
||||
const entryId = components.slice(-1)[0];
|
||||
|
||||
const title = components.slice(0, -1).reduce((accTitle, word, index) => `${accTitle}${index > 0 ? ' ' : ''}${word.slice(0, 1).toUpperCase()}${word.slice(1)}`, '');
|
||||
|
||||
return { title, entryId };
|
||||
}
|
||||
|
||||
function scrapeLatest(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElements = $('.site-list .scene-item').toArray();
|
||||
|
||||
return sceneElements.map((item) => {
|
||||
const element = $(item);
|
||||
|
||||
const sceneLinkElement = element.find('a').first();
|
||||
const { protocol, hostname, pathname } = new URL(sceneLinkElement.attr('href'));
|
||||
const url = `${protocol}//${hostname}${pathname}`;
|
||||
const { title, entryId } = titleExtractor(pathname);
|
||||
|
||||
const date = moment.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
|
||||
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
duration,
|
||||
rating: null,
|
||||
site,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElement = $('.scene-info');
|
||||
|
||||
const { protocol, hostname, pathname } = new URL(url);
|
||||
const originalUrl = `${protocol}//${hostname}${pathname}`;
|
||||
|
||||
const entryId = originalUrl.split('-').slice(-1)[0];
|
||||
const title = sceneElement.find('h1.scene-title.grey-text').text();
|
||||
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
|
||||
|
||||
const date = moment.utc(sceneElement.find('span.entry-date').text(), 'MMM D, YYYY').toDate();
|
||||
const actors = $('a.scene-title.grey-text.link').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;
|
||||
|
||||
const siteName = sceneElement.find('a.site-title').text();
|
||||
const siteId = siteName.replace(/[\s']+/g, '').toLowerCase();
|
||||
|
||||
const rawTags = $('.categories a.cat-tag').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
|
||||
const [channelSite, tags] = await Promise.all([
|
||||
knex('sites')
|
||||
.where({ id: siteId })
|
||||
.orWhere({ name: siteName })
|
||||
.first(),
|
||||
matchTags(rawTags),
|
||||
]);
|
||||
|
||||
return {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
description,
|
||||
actors,
|
||||
date,
|
||||
duration,
|
||||
tags,
|
||||
rating: null,
|
||||
site: channelSite || site,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await bhttp.get(`${site.url}?page=${page}`);
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await bhttp.get(url);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
};
|
||||
Loading…
Reference in New Issue