Displaying results in terminal table with select, scroll and experimental search. Expanded broken character handling for PervCity scraper.

This commit is contained in:
ThePendulum 2019-03-11 04:19:36 +01:00
parent 4fe7fd06e8
commit 02c421b737
9 changed files with 638 additions and 193 deletions

View File

@ -5,4 +5,27 @@ module.exports = {
'xempire', 'xempire',
'pervcity', 'pervcity',
], ],
columns: [
{
value: 'date',
format: 'MMM DD, YYYY',
width: 14,
},
{
value: 'site',
width: 20,
},
{
value: 'title',
width: 60,
},
{
value: 'actors',
width: 40,
},
{
value: 'rating',
width: 20,
},
],
}; };

View File

@ -9,21 +9,25 @@ module.exports = {
sites: { sites: {
hardx: { hardx: {
name: 'HardX', name: 'HardX',
label: 'HardX',
url: 'https://www.hardx.com', url: 'https://www.hardx.com',
description: 'Welcome to HardX.com, home of exclusive hardcore gonzo porn and first time anal scenes, DP, blowbangs and gangbangs from today\'s hottest porn stars!', description: 'Welcome to HardX.com, home of exclusive hardcore gonzo porn and first time anal scenes, DP, blowbangs and gangbangs from today\'s hottest porn stars!',
}, },
eroticax: { eroticax: {
name: 'EroticaX', name: 'EroticaX',
label: 'EroX',
url: 'https://www.eroticax.com', url: 'https://www.eroticax.com',
description: 'EroticaX.com features intimate scenes of passionate, erotic sex. Watch the sensual side of hardcore porn as your favorite pornstars have real, intense orgasms.', description: 'EroticaX.com features intimate scenes of passionate, erotic sex. Watch the sensual side of hardcore porn as your favorite pornstars have real, intense orgasms.',
}, },
darkx: { darkx: {
name: 'DarkX', name: 'DarkX',
label: 'DarkX',
url: 'https://www.darkx.com', url: 'https://www.darkx.com',
description: 'Watch interracial BBC porn videos on DarkX.com, featuring the best pornstars taking big black cock in exclusive scenes. The best black on white porn inside!', description: 'Watch interracial BBC porn videos on DarkX.com, featuring the best pornstars taking big black cock in exclusive scenes. The best black on white porn inside!',
}, },
lesbianx: { lesbianx: {
name: 'LesbianX', name: 'LesbianX',
label: 'LesX',
url: 'https://www.lesbianx.com', url: 'https://www.lesbianx.com',
description: 'LesbianX.com features today\'s top pornstars in hardcore lesbian porn. Watch passionate & intense girl on girl sex videos, from erotic kissing to pussy licking.', description: 'LesbianX.com features today\'s top pornstars in hardcore lesbian porn. Watch passionate & intense girl on girl sex videos, from erotic kissing to pussy licking.',
}, },
@ -36,6 +40,7 @@ module.exports = {
sites: { sites: {
analoverdose: { analoverdose: {
name: 'Anal Overdose', name: 'Anal Overdose',
label: 'AnalOD',
url: 'http://www.analoverdose.com', url: 'http://www.analoverdose.com',
description: 'Before proceeding, use caution: the stunning pornstars of Anal Overdose are so fiery that they cause heavy breathing, throbbing cocks and volcanic loads of cum. If you think you can handle the heat of smoking tits, sweltering pussy and red hot ass.', description: 'Before proceeding, use caution: the stunning pornstars of Anal Overdose are so fiery that they cause heavy breathing, throbbing cocks and volcanic loads of cum. If you think you can handle the heat of smoking tits, sweltering pussy and red hot ass.',
parameters: { parameters: {
@ -44,6 +49,7 @@ module.exports = {
}, },
bangingbeauties: { bangingbeauties: {
name: 'Banging Beauties', name: 'Banging Beauties',
label: 'BBeaus',
url: 'http://www.bangingbeauties.com', url: 'http://www.bangingbeauties.com',
description: 'Banging Beauties isn\'t just a porn site; it\'s the gateway to all your pussy-obsessed fantasies! Our members\' area is flowing with beautiful pornstars anticipating big dick throbbing in their syrupy pink slits. These experienced babes love brutal vaginal pounding! Similarly, they\'re eager for anal switch-hitting to shake things up. However, it\'s not only about gorgeous sexperts filling their hungry holes. Sometimes, it\'s all about innocent rookies earning their pornstar status in first time threesomes and premier interracial scenes.', description: 'Banging Beauties isn\'t just a porn site; it\'s the gateway to all your pussy-obsessed fantasies! Our members\' area is flowing with beautiful pornstars anticipating big dick throbbing in their syrupy pink slits. These experienced babes love brutal vaginal pounding! Similarly, they\'re eager for anal switch-hitting to shake things up. However, it\'s not only about gorgeous sexperts filling their hungry holes. Sometimes, it\'s all about innocent rookies earning their pornstar status in first time threesomes and premier interracial scenes.',
parameters: { parameters: {
@ -52,6 +58,7 @@ module.exports = {
}, },
oraloverdose: { oraloverdose: {
name: 'Oral Overdose', name: 'Oral Overdose',
label: 'OralOD',
url: 'http://www.oraloverdose.com', url: 'http://www.oraloverdose.com',
description: 'Oral Overdose is the only site you need to live out every saliva soaked blowjob of your dreams in HD POV! We\'ve got the most stunning cocksuckers in the world going to town on big dick. These babes not only love cock, they can\'t get enough of it! In fact, there is no prick too huge for our hungry girls\' throats. You\'ll find gorgeous, big tits pornstars exercising their gag reflex in intense balls deep facefuck scenes. We also feature fresh, young newbies taking on the gagging deepthroat challenge.', description: 'Oral Overdose is the only site you need to live out every saliva soaked blowjob of your dreams in HD POV! We\'ve got the most stunning cocksuckers in the world going to town on big dick. These babes not only love cock, they can\'t get enough of it! In fact, there is no prick too huge for our hungry girls\' throats. You\'ll find gorgeous, big tits pornstars exercising their gag reflex in intense balls deep facefuck scenes. We also feature fresh, young newbies taking on the gagging deepthroat challenge.',
parameters: { parameters: {
@ -60,6 +67,7 @@ module.exports = {
}, },
chocolatebjs: { chocolatebjs: {
name: 'Chocolate BJs', name: 'Chocolate BJs',
label: 'ChocBJ',
url: 'http://www.chocolatebjs.com', url: 'http://www.chocolatebjs.com',
description: 'You\'ve just won the golden ticket to the best Chocolate BJs on the planet! We\'ve sought far and wide to bring you the most beautiful black and ethnic pornstars. And they\'re in our members\' area now! They can\'t wait to suck your white lollipop and lick the thick cream shooting from your big dick. Of course, no matter how sweet the booty or juicy the big tits, these brown foxes aren\'t all sugar and spice. In fact, when it comes to giving head, these big ass ebony babes know what they want: huge white cocks filling their throats!', description: 'You\'ve just won the golden ticket to the best Chocolate BJs on the planet! We\'ve sought far and wide to bring you the most beautiful black and ethnic pornstars. And they\'re in our members\' area now! They can\'t wait to suck your white lollipop and lick the thick cream shooting from your big dick. Of course, no matter how sweet the booty or juicy the big tits, these brown foxes aren\'t all sugar and spice. In fact, when it comes to giving head, these big ass ebony babes know what they want: huge white cocks filling their throats!',
parameters: { parameters: {
@ -68,6 +76,7 @@ module.exports = {
}, },
upherasshole: { upherasshole: {
name: 'Up Her Asshole', name: 'Up Her Asshole',
label: 'UpHerA',
url: 'http://www.upherasshole.com', url: 'http://www.upherasshole.com',
description: 'You don\'t need to travel the globe in search of the anal wonders of the world, because you get your own private tour right here on Up Her Asshole! Our stunning pornstars and rookie starlets welcome all ass fetish and anal sex fans, with their twerking bubble butts and winking assholes. However, big booty worship is just a slice of the fun. Combined with juicy tits (big and small), wet pussy (hairy and bald), these girls deliver a spectacular sensory experience in HD POV. Not only are you in danger of busting a nut before the going gets good, but also when the good turns remarkable with rimming, fingering and butt toys!', description: 'You don\'t need to travel the globe in search of the anal wonders of the world, because you get your own private tour right here on Up Her Asshole! Our stunning pornstars and rookie starlets welcome all ass fetish and anal sex fans, with their twerking bubble butts and winking assholes. However, big booty worship is just a slice of the fun. Combined with juicy tits (big and small), wet pussy (hairy and bald), these girls deliver a spectacular sensory experience in HD POV. Not only are you in danger of busting a nut before the going gets good, but also when the good turns remarkable with rimming, fingering and butt toys!',
parameters: { parameters: {

617
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -36,6 +36,8 @@
"cheerio": "^1.0.0-rc.2", "cheerio": "^1.0.0-rc.2",
"config": "^3.0.1", "config": "^3.0.1",
"moment": "^2.24.0", "moment": "^2.24.0",
"terminal-kit": "^1.27.0" "neo-blessed": "^0.2.0",
"opn": "^5.4.0",
"tty-table": "^2.7.0"
} }
} }

View File

@ -1,11 +1,29 @@
'use strict'; 'use strict';
const config = require('config'); const config = require('config');
const { terminal } = require('terminal-kit');
const moment = require('moment'); const moment = require('moment');
const blessed = require('neo-blessed');
const networks = require('../networks.js'); const networks = require('../networks.js');
const scrapers = require('./scrapers'); const scrapers = require('./scrapers');
const render = require('./tui/render');
function initScreen() {
const screen = blessed.screen({
title: `traxxx ${new Date().getTime()}`,
smartCSR: true,
mouse: false,
});
screen.enableInput();
screen.key(['escape', 'q', 'C-c'], () => {
screen.render();
screen.destroy();
});
return screen;
}
function accumulateIncludedSites() { function accumulateIncludedSites() {
return config.include.reduce((acc, network) => { return config.include.reduce((acc, network) => {
@ -83,7 +101,7 @@ function accumulateSites() {
} }
async function fetchScenes(sites) { async function fetchScenes(sites) {
return Promise.all(sites.map(async (site) => { const scenesPerSite = await Promise.all(sites.map(async (site) => {
const scraper = scrapers[site.id] || scrapers[site.network]; const scraper = scrapers[site.id] || scrapers[site.network];
if (scraper) { if (scraper) {
@ -92,25 +110,18 @@ async function fetchScenes(sites) {
return []; return [];
})); }));
}
function exit() { return scenesPerSite.reduce((acc, siteScenes) => ([...acc, ...siteScenes]), []);
terminal.grabInput(false);
terminal.clear();
terminal.processExit();
} }
async function init() { async function init() {
const screen = initScreen();
const sites = accumulateSites(); const sites = accumulateSites();
const scenes = await fetchScenes(sites); const scenes = await fetchScenes(sites);
const sortedScenes = scenes.sort(({ date: dateA }, { date: dateB }) => moment(dateB).diff(dateA));
terminal.singleColumnMenu(scenes[0].latest.map(scene => `[${scene.siteId} ${moment(scene.date).format('YYYY-MM-DD')}] ${scene.title} (${scene.actors.join(', ')}) ★ ${scene.rating.stars.toFixed(2)}`)); render(sortedScenes, screen);
terminal.on('key', (name) => {
if (name === 'CTRL_C') {
exit();
}
});
} }
init(); init();

View File

@ -9,9 +9,9 @@ function scrape(html, site) {
const sceneLinkElement = $('#scene_title_border a'); const sceneLinkElement = $('#scene_title_border a');
const url = `${site.url}/${sceneLinkElement.attr('href')}`; const url = `${site.url}/${sceneLinkElement.attr('href')}`;
const title = sceneLinkElement.attr('title'); const title = sceneLinkElement.attr('title').replace(/\u00E2\u0080\u0099/g, '\''); // replace weird apostrophes
const actors = $('.home_model_name a').toArray().map(element => $(element).text().replace(/,[\u0020\u00A0\u202F]/, '')); const actors = $('.home_model_name a').toArray().map(element => $(element).text().replace(/,[\u0020\u00A0\u202F]/, '')); // replace weird commas
const date = moment.utc($('.add_date').text(), 'DD-MM-YYYY').toDate(); const date = moment.utc($('.add_date').text(), 'DD-MM-YYYY').toDate();
const stars = $('img[src*="/star.png"]').toArray().map(element => $(element).attr('src')).length || null; const stars = $('img[src*="/star.png"]').toArray().map(element => $(element).attr('src')).length || null;
@ -26,11 +26,10 @@ function scrape(html, site) {
dislikes: null, dislikes: null,
stars, stars,
}, },
siteId: site.id, site,
}; };
} }
async function fetchReleases(site) { async function fetchReleases(site) {
// const session = bhttp.session(); // const session = bhttp.session();
@ -39,9 +38,7 @@ async function fetchReleases(site) {
const latest = elements.total_arr.map(html => scrape(html, site)); const latest = elements.total_arr.map(html => scrape(html, site));
return { return latest;
latest,
};
} }
module.exports = fetchReleases; module.exports = fetchReleases;

View File

@ -16,7 +16,7 @@ function scrape(html, site) {
const [likes, dislikes] = $(element).find('.value') const [likes, dislikes] = $(element).find('.value')
.toArray() .toArray()
.map(value => Number($(value).text())); .map(value => Number($(value).text()));
const stars = Math.floor(((likes * 5 + dislikes) / (likes + dislikes)) * 100) / 100; const stars = likes || dislikes ? Math.floor(((likes * 5 + dislikes) / (likes + dislikes)) * 100) / 100 : null;
const actors = $(element).find('.sceneActors a') const actors = $(element).find('.sceneActors a')
.map((actorIndex, actorElement) => $(actorElement).attr('title')) .map((actorIndex, actorElement) => $(actorElement).attr('title'))
@ -34,7 +34,7 @@ function scrape(html, site) {
dislikes, dislikes,
stars, stars,
}, },
siteId: site.id, site,
}; };
}); });
} }
@ -45,10 +45,10 @@ async function fetchReleases(site) {
bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`), bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`),
]); ]);
return { return [
latest: scrape(latestRes.body.toString(), site), ...scrape(upcomingRes.body.toString(), site, true),
upcoming: scrape(upcomingRes.body.toString(), site), ...scrape(latestRes.body.toString(), site),
}; ];
} }
module.exports = fetchReleases; module.exports = fetchReleases;

22
src/tui/formatters.js Normal file
View File

@ -0,0 +1,22 @@
'use strict';
const moment = require('moment');
const formatters = {
site: site => site.name,
date: (date, column) => moment(date).format(column.format || 'MMM DD, YYYY'),
actors: actors => actors.join(', '),
rating: (rating) => {
if (rating.stars === null) {
return 'Unrated';
}
if (rating.likes === null || rating.dislikes === null) {
return `${rating.stars.toFixed(2)}`;
}
return `${rating.stars.toFixed(2)}${String(rating.likes).padEnd(3)}${String(rating.dislikes).padEnd(3)}`;
},
};
module.exports = formatters;

94
src/tui/render.js Normal file
View File

@ -0,0 +1,94 @@
'use strict';
const config = require('config');
const blessed = require('neo-blessed');
const moment = require('moment');
const opn = require('opn');
const formatters = require('./formatters');
function render(scenes, screen) {
const tableTop = blessed.Text({
content: config.columns.reduce((acc, column, index) => `${acc}${'─'.repeat(column.width)}${index < config.columns.length - 1 ? '┬' : '┐\x1b[0m'}`, '\x1b[30m┌'),
});
const items = scenes.map((scene, sceneIndex) => {
const isFuture = moment(scene.date).isAfter();
const row = config.columns.reduce((acc, column) => {
const value = (formatters[column.value]
? formatters[column.value](scene[column.value], column)
: scene[column.value])
.toString();
const truncatedValue = value.length > column.width - 2 ? `${value.slice(0, column.width - 2 - 3)}...` : value;
const paddedValue = truncatedValue.padEnd(column.width - 1).padStart(column.width);
const coloredValue = isFuture ? `\x1b[92m${paddedValue}\x1b[0m` : `\x1b[97m${paddedValue}\x1b[0m`;
return `${acc}${coloredValue}\x1b[90m│\x1b[0m`;
}, '\x1b[90m│\x1b[0m');
if (sceneIndex < scenes.length - 1) {
const line = config.columns.reduce((acc, column, index) => `${acc}${'─'.repeat(column.width)}${index < config.columns.length - 1 ? '┼' : '┤\x1b[0m'}`, '\n\x1b[30m├');
return `${row}${line}`;
}
return `${row}${sceneIndex}`;
});
function search(cb) {
const searchbox = blessed.Textbox({
inputOnFocus: true,
});
screen.append(searchbox);
searchbox.focus();
screen.render();
searchbox.on('submit', () => {
menu.focus();
cb(null, searchbox.value);
screen.append(menu);
screen.render();
});
}
const menu = blessed.List({
style: {
selected: {
bold: true,
},
},
top: 1,
height: screen.rows - 3,
keys: true,
vi: true,
mouse: true,
search,
items,
});
const tableBottom = blessed.Text({
content: config.columns.reduce((acc, column, index) => `${acc}${'─'.repeat(column.width)}${index < config.columns.length - 1 ? '┴' : '┘\x1b[0m\n'}`, '\x1b[30m└'),
top: screen.rows - 2,
});
screen.append(tableTop);
screen.append(menu);
screen.append(tableBottom);
menu.focus();
menu.on('select', (child) => {
const scene = scenes[menu.getItemIndex(child)];
opn(scene.url);
});
screen.render();
}
module.exports = render;