DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added tag groups. Added MOFOS scraped. Improved entry ID handling.

This commit is contained in:
ThePendulum 2019-04-07 02:15:57 +02:00
parent 6fea65a411
commit 885a815022
7 changed files with 408 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
migrations/20190325001339_releases.js
View File

@ -19,10 +19,20 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('directors');
}))
.then(() => knex.schema.createTable('tags_groups', (table) => {
table.string('group', 20)
.primary();
table.string('name', 20);
}))
.then(() => knex.schema.createTable('tags', (table) => {
table.string('tag', 20)
.primary();
table.string('group_id', 20)
.references('group')
.inTable('tags_groups');
table.string('alias_for', 20)
.references('tag')
.inTable('tags');
@ -121,4 +131,5 @@ exports.down = knex => Promise.resolve()
.then(() => knex.schema.dropTable('networks'))
.then(() => knex.schema.dropTable('actors'))
.then(() => knex.schema.dropTable('directors'))
.then(() => knex.schema.dropTable('tags_groups'))
.then(() => knex.schema.dropTable('tags'));

6
seeds/networks.js
View File

@ -33,6 +33,12 @@ exports.seed = knex => Promise.resolve()
url: 'https://www.legalporno.com',
description: 'The Best HD Porn For You!',
},
{
id: 'mofos',
name: 'MOFOS',
url: 'https://www.mofos.com',
description: 'Check out the Official Mofos Network of best amateur pornsites. Girlfriend – voyeur - college girls - first anal & more. Bonus Milf sites for wifey lovers.',
},
{
id: 'pervcity',
name: 'Perv City',

106
seeds/sites.js
View File

@ -577,6 +577,112 @@ exports.seed = knex => Promise.resolve()
description: 'The Best HD Porn For You!',
network_id: 'legalporno',
},
// MOFOS
{
id: 'girlsgonepink',
name: 'Girls Gone Pink',
url: 'https://www.mofos.com/tour/videos/girls-gone-pink',
description: "There comes a point in every woman's life when she gets a little curious about what some hot girl on girl sex could be like. Whether they're lesbian or just straight and incredibly daring and open-minded, the end result is the same. GirlsGonePink.com is full of soft lips, long flowing hair, and sensual feminine figures that are enough to get any horny minx's blood pumping. Premium full-length lesbian porn videos await you full of perfect boobs, pointy nipples, round butts, and luscious legs that usually stay separated!",
network_id: 'mofos',
},
{
id: 'ebonysextapes',
name: 'Ebony Sex Tapes',
url: 'https://www.mofos.com/tour/videos/ebony-sex-tapes',
description: 'Once you go black, you never go back! Did you think that was only how white women feel about black men? Well if you did, that can only mean you never had a stacked, curvy, big ass, beautiful black teen riding your hard white cock. Watch these lucky guys fuck their Ebony Girlfriends at EbonySexTapes.com.',
network_id: 'mofos',
},
{
id: 'sharemybf',
name: 'Share My BF',
url: 'https://www.mofos.com/tour/videos/share-my-bf',
description: 'Would your cock be able to handle 2 wet pussies at the same time? One hot teen riding your face while the other deepthroats you. You know your GF tells all her friends how big your dick is. Imagine if you can fuck her and her friend at the same time? Live the fantasy at ShareMyBF.com.',
network_id: 'mofos',
},
{
id: 'dontbreakme',
name: "Don't Break Me",
url: 'https://www.mofos.com/tour/videos/dont-break-me',
description: 'DontBreakMe.com is about tiny spinners fucking big guys with massive dicks! Most of these chicks are shorter than 5 feet tall and weigh less than 100lbs. Meanwhile, these girls are paired up with guys who tower over them by at least 1.5 feet and have 9" dicks!! The look on their faces when they see that huge dick pop out of his pants is priceless. While it turns them on they usually get a bit nervous: "how will I squeeze that huge cock inside?" Ouch! Check it out.',
network_id: 'mofos',
},
{
id: 'iknowthatgirl',
name: 'I Know That Girl',
url: 'https://www.mofos.com/tour/videos/i-know-that-girl',
description: 'Every single gorgeous girl you see on this site is 100% Real! They are all part of the biggest user submitted, amateur video site in the world...IKnowThatGirl.com! Hot young girlfriends getting kinky on camera, sucking and fucking, even stuffing dildos up their tight pussies, all filmed on home video and leaked to us by some lowlife, soon to be ex-boyfriend or former best friend! Oh well... Enjoy!',
network_id: 'mofos',
},
{
id: 'letstryanal',
name: 'Lets Try Anal',
url: 'https://www.mofos.com/tour/videos/lets-try-anal',
description: "This isn't just another anal site! Letstryanal.com features the hottest real footage of amateur girls and their first time ass fucking experiences. Watch it all... innocent girlfriends being convinced to try anal, their faces of pain and screaming as they beg their boyfriend to \"please go slower\" while a large cock penetrates their tight asses for the first time! Let's face it, there is nothing like seeing a cock disappear in a virgin asshole. It's so hot!",
network_id: 'mofos',
},
{
id: 'latinasextapes',
name: 'Latina Sex Tapes',
url: 'https://www.mofos.com/tour/videos/latina-sex-tapes',
description: "100% Real Latina Girls getting fucked by their boyfriends, filmed and submitted to us for Big $$$! Watch amazing real footage and private videos of these beautiful amateur girls, their perfectly tanned bodies, mouth-watering curves, luscious round asses, and mind blowing accents! We've only kept the best, most outstanding sex videos and uploaded them for you to watch. You'll be amazed with what we received, and more is on the way!",
network_id: 'mofos',
},
{
id: 'publicpickups',
name: 'Public Pickups',
url: 'https://www.mofos.com/tour/videos/public-pickups',
description: "Check out the hottest REAL footage of young girls getting picked up and fucked in public! The girls are usually shy around guys approaching them with a video camera, but that's the fun part. Besides their shyness slowly disappears after they're offered money to get dirty. While it's a real turn on seeing the girls flash and get fondled in public... the hottest part is watching them get fucked everywhere...in cars, parks, clubs, even the library!",
network_id: 'mofos',
},
{
id: 'pervsonpatrol',
name: 'Pervs On Patrol',
url: 'https://www.mofos.com/tour/videos/pervs-on-patrol',
description: "A while back, this beautiful girl who lived next door use to always undress with her window opened. This girl had no fucking clue that I was jerking off over her from across the yard. One day I decided to grab my dad's camera and start filming her. It was amazing... until she finally caught me. Fuck, this girl was PISSED!..., but could you fucking believe that once she calmed down she was actually a little turned on by the whole situation,... and what happened next changed my life!",
network_id: 'mofos',
},
{
id: 'strandedteens',
name: 'Stranded Teens',
url: 'https://www.mofos.com/tour/videos/stranded-teens',
description: "Watch videos on StrandedTeens.com and you will never look at a hitchhiker the same way again! Some of these girls will do anything for a ride or simply to make a friend - even the shy ones. From giving road head to getting ass-fucked on the hood of the car, you can watch it all. Check it out now, you won't be disappointed!",
network_id: 'mofos',
},
{
id: 'realslutparty',
name: 'Real Slut Party',
url: 'https://www.mofos.com/tour/videos/real-slut-party',
description: "Wanna see the most mind blowing college sex parties from across the country? It's the real deal, all caught on video and submitted by you! Insane college craziness, pussy packed house parties, holiday orgies, backyard BBQ's gone wrong and hundreds of tight, young girls getting crazy, stripped down, and on the prowl for all the cock they can find!",
network_id: 'mofos',
},
{
id: 'mofoslab',
name: 'MOFOS Lab',
url: 'https://www.mofos.com/tour/videos/mofos-lab',
description: "We've received your feedback and are experimenting with turning your wildest fantasies into the ultimate POV experience; this is Mofos Lab! Featuring today's hottest and freshest talent, immerse yourself in an exciting Mofos venture that brings you the edgiest new content!",
network_id: 'mofos',
},
{
id: 'mofosbsides',
name: 'Mofos B Sides',
url: 'https://www.mofos.com/tour/videos/mofos-b-sides',
description: "Mofos B-Sides is a doorway to new, unseen amateur video! Hundreds of clips have been submitted to Mofos through the years and we've never shown them to you until now. We'll give you a little bit at a time, from random girls in random scenario\\’s, and maybe even an occasional free video from our friends at Brazzers, Twisty's and Babes! Check it all out and let us know what you think.",
network_id: 'mofos',
},
{
id: 'shesafreak',
name: "She's A Freak",
url: 'https://www.mofos.com/tour/videos/shes-a-freak',
description: "Fresh, young amateur girls with beautiful tight bodies, pushing themselves to the limit! It's just another great way that today's hottest new models are choosing to showcase their stunning bodies and show all of us that they're ready for more! Soaking wet masturbation, fisting, squirting, double penetration and anal toys are just some of the things they do to show us how freaky they can be and how ready they are to graduate from toys to thick, fat cock!",
network_id: 'mofos',
},
{
id: 'blogs',
name: 'Blogs',
url: 'https://www.mofos.com/tour/videos/blog',
description: 'Watch intimate video diaries from hot chicks who like nothing better than to show and tell, whether it be their bangin’ new boobs or their sweet little pussies. These are real life confessions, from real life sluts, who want you to know all their dirtiest secrets.',
network_id: 'mofos',
},
// PERVCITY
{
id: 'analoverdose',

184
seeds/tags.js
View File

@ -3,14 +3,59 @@
/* eslint-disable max-len */
exports.seed = knex => Promise.resolve()
.then(() => knex('tags').del())
.then(() => knex('tags_groups').insert([
{
group: 'penetration',
name: 'Penetration',
},
{
group: 'position',
name: 'Position',
},
{
group: 'group',
name: 'Group sex',
},
{
group: 'location',
name: 'Location',
},
{
group: 'body',
name: 'Body',
},
{
group: 'age',
name: 'Age',
},
{
group: 'hair',
name: 'Hair',
},
{
group: 'ethnicity',
name: 'Ethnicity',
},
{
group: 'clothing',
name: 'Clothing',
},
]))
.then(() => knex('tags').insert([
{
tag: '69',
alias_for: null,
group_id: 'position',
},
{
tag: 'airtight',
alias_for: null,
group_id: 'penetration',
},
{
tag: 'anal sex',
alias_for: null,
group_id: null,
},
{
tag: 'anal fingering',
@ -25,12 +70,13 @@ exports.seed = knex => Promise.resolve()
alias_for: null,
},
{
tag: 'anal toy',
tag: 'anal toys',
alias_for: null,
},
{
tag: 'asian',
alias_for: null,
group_id: 'ethnicity',
},
{
tag: 'ass licking',
@ -39,11 +85,17 @@ exports.seed = knex => Promise.resolve()
{
tag: 'athletic',
alias_for: null,
group_id: 'body',
},
{
tag: 'ATM',
alias_for: null,
},
{
tag: 'bathroom',
alias_for: null,
group_id: 'location',
},
{
tag: 'BDSM',
alias_for: null,
@ -51,22 +103,32 @@ exports.seed = knex => Promise.resolve()
{
tag: 'BBC',
alias_for: null,
group_id: 'body',
},
{
tag: 'big cock',
alias_for: null,
group_id: 'body',
},
{
tag: 'big butt',
alias_for: null,
group_id: 'body',
},
{
tag: 'big boobs',
alias_for: null,
group_id: 'body',
},
{
tag: 'blonde',
tag: 'black hair',
alias_for: null,
group_id: 'hair',
},
{
tag: 'blonde hair',
alias_for: null,
group_id: 'hair',
},
{
tag: 'blowjob',
@ -75,14 +137,16 @@ exports.seed = knex => Promise.resolve()
{
tag: 'blowbang',
alias_for: null,
group_id: 'group',
},
{
tag: 'bondage',
alias_for: null,
},
{
tag: 'brunette',
tag: 'brown hair',
alias_for: null,
group_id: 'hair',
},
{
tag: 'bukkake',
@ -100,6 +164,10 @@ exports.seed = knex => Promise.resolve()
tag: 'corporal punishment',
alias_for: null,
},
{
tag: 'couples',
alias_for: null,
},
{
tag: 'cowgirl',
alias_for: null,
@ -147,6 +215,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'ebony',
alias_for: null,
group_id: 'ethnicity',
},
{
tag: 'electric shock',
@ -191,6 +260,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'FMF',
alias_for: null,
group_id: 'group',
},
{
tag: 'gag',
@ -199,6 +269,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'gangbang',
alias_for: null,
group_id: 'group',
},
{
tag: 'gaping',
@ -211,6 +282,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'hairy',
alias_for: null,
group_id: 'body',
},
{
tag: 'hardcore',
@ -219,6 +291,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'high heels',
alias_for: null,
group_id: 'clothing',
},
{
tag: 'humiliation',
@ -251,22 +324,27 @@ exports.seed = knex => Promise.resolve()
{
tag: 'lingerie',
alias_for: null,
group_id: 'clothing',
},
{
tag: 'MILF',
alias_for: null,
group_id: 'age',
},
{
tag: 'MFM',
alias_for: null,
group_id: 'group',
},
{
tag: 'missionary',
alias_for: null,
group_id: 'position',
},
{
tag: 'natural',
alias_for: null,
group_id: 'body',
},
{
tag: 'nipple clamps',
@ -276,6 +354,16 @@ exports.seed = knex => Promise.resolve()
tag: 'oral creampie',
alias_for: null,
},
{
tag: 'orgy',
alias_for: null,
group_id: 'group',
},
{
tag: 'outdoors',
alias_for: null,
group_id: 'location',
},
{
tag: 'pain',
alias_for: null,
@ -292,13 +380,18 @@ exports.seed = knex => Promise.resolve()
tag: 'piercings',
alias_for: null,
},
{
tag: 'pov',
alias_for: null,
},
{
tag: 'pussy eating',
alias_for: null,
},
{
tag: 'redhead',
tag: 'red hair',
alias_for: null,
group_id: 'hair',
},
{
tag: 'reverse cowgirl',
@ -315,6 +408,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'Russian',
alias_for: null,
group_id: 'ethnicity',
},
{
tag: 'saliva',
@ -359,6 +453,7 @@ exports.seed = knex => Promise.resolve()
{
tag: 'stockings',
alias_for: null,
group_id: 'clothing',
},
{
tag: 'strap-on dildo',
@ -375,17 +470,20 @@ exports.seed = knex => Promise.resolve()
{
tag: 'tattoo',
alias_for: null,
group_id: 'body',
},
{
tag: 'threesome',
alias_for: null,
group_id: 'group',
},
{
tag: 'teen',
alias_for: null,
group_id: 'age',
},
{
tag: 'toy',
tag: 'toys',
alias_for: null,
},
{
@ -408,14 +506,24 @@ exports.seed = knex => Promise.resolve()
tag: 'voyeur',
alias_for: null,
},
{
tag: 'wet',
alias_for: null,
},
{
tag: 'white',
alias_for: null,
group_id: 'ethnicity',
},
{
tag: 'wife',
alias_for: null,
},
{
tag: 'office',
alias_for: null,
group_id: 'location',
},
]))
.then(() => knex('tags').insert([
// ALIASES
@ -479,9 +587,13 @@ exports.seed = knex => Promise.resolve()
tag: 'black',
alias_for: 'ebony',
},
{
tag: 'blonde',
alias_for: 'blonde hair',
},
{
tag: 'blondes',
alias_for: 'blonde',
alias_for: 'blonde hair',
},
{
tag: 'blowjobs',
@ -491,6 +603,10 @@ exports.seed = knex => Promise.resolve()
tag: 'blowjob (double)',
alias_for: 'double blowjob',
},
{
tag: 'blowjob (pov)',
alias_for: 'blowjob',
},
{
tag: 'boob job',
alias_for: 'enhanced boobs',
@ -499,13 +615,17 @@ exports.seed = knex => Promise.resolve()
tag: 'boobjob',
alias_for: 'enhanced boobs',
},
{
tag: 'brunette',
alias_for: 'brown hair',
},
{
tag: 'brunettes',
alias_for: 'brunette',
alias_for: 'brown hair',
},
{
tag: 'buttplug',
alias_for: 'anal toy',
alias_for: 'anal toys',
},
{
tag: 'caning',
@ -523,6 +643,10 @@ exports.seed = knex => Promise.resolve()
tag: 'clover clamps',
alias_for: 'nipple clamps',
},
{
tag: 'couples fantasies',
alias_for: 'couples',
},
{
tag: 'creampies',
alias_for: 'creampie',
@ -553,7 +677,7 @@ exports.seed = knex => Promise.resolve()
},
{
tag: 'dildo',
alias_for: 'toy',
alias_for: 'toys',
},
{
tag: 'doggystyle',
@ -563,6 +687,10 @@ exports.seed = knex => Promise.resolve()
tag: 'doggie style',
alias_for: 'doggy style',
},
{
tag: 'doggystyle (standing)',
alias_for: 'standing doggy style',
},
{
tag: 'dom',
alias_for: 'BDSM',
@ -631,6 +759,10 @@ exports.seed = knex => Promise.resolve()
tag: 'enhanced',
alias_for: 'enhanced boobs',
},
{
tag: 'face sitting',
alias_for: 'facesitting',
},
{
tag: 'facials',
alias_for: 'facial',
@ -711,9 +843,21 @@ exports.seed = knex => Promise.resolve()
tag: 'prolapsing',
alias_for: 'anal prolapse',
},
{
tag: 'raven',
alias_for: 'black hair',
},
{
tag: 'raven hair',
alias_for: 'black hair',
},
{
tag: 'redhead',
alias_for: 'red hair',
},
{
tag: 'red head',
alias_for: 'redhead',
alias_for: 'red hair',
},
{
tag: 'rimming',
@ -747,6 +891,14 @@ exports.seed = knex => Promise.resolve()
tag: 'sadism',
alias_for: 'BDSM',
},
{
tag: 'scissoring',
alias_for: 'lesbian',
},
{
tag: 'sex toys',
alias_for: 'toys',
},
{
tag: 'shaved pussy',
alias_for: 'shaved',
@ -811,10 +963,6 @@ exports.seed = knex => Promise.resolve()
tag: 'tiny tits',
alias_for: 'small boobs',
},
{
tag: 'toys',
alias_for: 'toy',
},
{
tag: 'trimmed pussy',
alias_for: 'trimmed',
@ -827,6 +975,14 @@ exports.seed = knex => Promise.resolve()
tag: 'whipping',
alias_for: 'corporal punishment',
},
{
tag: 'work',
alias_for: 'office',
},
{
tag: 'workplace',
alias_for: 'office',
},
{
tag: 'zapper',
alias_for: 'electric shock',

13
src/fetch-releases.js
View File

@ -58,18 +58,18 @@ async function accumulateIncludedSites() {
async function findDuplicateReleases(latestReleases, _siteId) {
const latestReleasesShootIds = latestReleases.map(release => release.shootId).filter(release => release !== undefined);
const latestReleasesPageIds = latestReleases.map(release => release.pageId).filter(release => release !== undefined);
const latestReleasesEntryIds = latestReleases.map(release => release.entryId).filter(release => release !== undefined);
return knex('releases')
.whereIn('shoot_id', latestReleasesShootIds)
.orWhereIn('shoot_id', latestReleasesPageIds);
.orWhereIn('entry_id', latestReleasesEntryIds);
}
async function storeReleases(releases) {
const curatedReleases = releases.map(release => ({
site_id: release.site.id,
shoot_id: release.shootId || null,
entry_id: release.entry_id || null,
entry_id: release.entryId || null,
url: release.url,
title: release.title,
date: release.date,
@ -97,13 +97,16 @@ async function fetchNewReleases(scraper, site, afterDate, accReleases = [], page
const latestReleases = await scraper.fetchLatest(site, page);
const duplicateReleases = await findDuplicateReleases(latestReleases, site.id);
const duplicateReleasesShootIds = new Set(
const duplicateReleasesIds = new Set(
duplicateReleases
.map(release => release.shoot_id)
// exclude accumulated releases to prevent an infinite loop if the next page contains the same releases as the previous
.concat(duplicateReleases.map(release => release.entry_id))
.concat(accReleases.map(release => release.shootId)),
);
const uniqueReleases = latestReleases.filter(release => !duplicateReleasesShootIds.has(String(release.shootId)) && moment(release.date).isAfter(afterDate));
const uniqueReleases = latestReleases.filter(release => !duplicateReleasesIds.has(String(release.shootId))
&& !duplicateReleasesIds.has(String(release.entryId))
&& moment(release.date).isAfter(afterDate));
console.log(`${site.name}: Scraped page ${page}, ${uniqueReleases.length} unique releases`);

2
src/scrapers/index.js
View File

@ -5,6 +5,7 @@ const brazzers = require('./brazzers');
const julesjordan = require('./julesjordan');
const kink = require('./kink');
const legalporno = require('./legalporno');
const mofos = require('./mofos');
const pervcity = require('./pervcity');
const privateNetwork = require('./private'); // reserved keyword
const vixen = require('./vixen');
@ -16,6 +17,7 @@ module.exports = {
julesjordan,
kink,
legalporno,
mofos,
pervcity,
private: privateNetwork,
vixen,

105
src/scrapers/mofos.js Normal file
View File

@ -0,0 +1,105 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.widget-release-card').toArray();
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.title a');
const title = sceneLinkElement.text().trim();
const url = `https://www.mofos.com${sceneLinkElement.attr('href')}`;
const entryId = url.split('/').slice(-2, -1)[0];
const date = moment.utc($(element).find('.date-added').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.girls-name a').map((actorIndex, actorElement) => $(actorElement).attr('title').replace(/\s+/g, ' ')).toArray();
const stars = Number($(element).find('.rating').text().slice(0, -1).trim()) / 20;
return {
url,
entryId,
title,
actors,
date,
rating: {
stars,
},
site,
};
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElement = $('.video-info');
const entryId = url.split('/').slice(-2, -1)[0];
const title = sceneElement.find('.title').text();
const description = sceneElement.find('.desc').text();
const actors = sceneElement.find('.girls-site-box a.model-name').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const siteElement = sceneElement.find('.site-name');
const sitename = siteElement.text().trim();
const siteId = sitename.replace(/\s+/g, '').toLowerCase();
const siteUrl = siteElement.attr('href').split('/').slice(0, 4).join('/');
const stars = Number(sceneElement.find('.rating-box .rating').text().slice(0, -1).trim()) / 20;
const rawTags = sceneElement.find('.categories a').map((tagIndex, tagElement) => $(tagElement).text().trim()).toArray();
const [channelSite, tags] = await Promise.all([
knex('sites')
.where({ id: siteId })
.orWhere({ url: `https://www.mofos.com${siteUrl}` })
.orWhere({ name: sitename })
.first(),
matchTags(rawTags),
]);
return {
url,
entryId,
title,
actors,
tags,
rating: {
stars,
},
site,
};
}
async function fetchLatest(site, page = 1) {
const res = page > 1
? await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/${page}/`)
: await bhttp.get(`${site.url}/all-models/all-categories/alltime/bydate/`); // explicit page 1 redirects to homepage
return scrape(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const res = await bhttp.get(`${site.url}/all-models/all-categories/upcoming/bydate/`);
return scrape(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(url);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchUpcoming,
fetchScene,
};