Replaced network and tag files with SQLite database.

This commit is contained in:
2019-03-25 03:57:33 +01:00
parent f44c5083ff
commit 4d7f323e86
22 changed files with 1948 additions and 654 deletions

View File

@@ -4,26 +4,7 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const tagMap = {
Anal: 'anal',
Asian: 'asian',
'Ass To Mouth': 'ATM',
'Big Cocks': 'big cock',
Black: 'BBC',
Blondes: 'blonde',
Brunettes: 'brunette',
Blowjobs: 'blowjob',
Creampie: 'creampie',
'Deep Throat': 'deepthroat',
Facial: 'facial',
Interracial: 'interracial',
Lingerie: 'lingerie',
Natural: 'natural',
'Red Head': 'readhead',
'School Girl': 'schoolgirl',
Tattoo: 'tattoo',
Teen: 'teen',
};
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
@@ -87,7 +68,7 @@ function scrapeUpcoming(html, site) {
});
}
function scrapeScene(html, url, site) {
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const title = $('.title_bar_hilite').text();
@@ -104,7 +85,7 @@ function scrapeScene(html, url, site) {
const stars = Number($('.avg_rating').text().trim().replace(/[\s|Avg Rating:]/g, ''));
const rawTags = $('.update_tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const tags = rawTags.reduce((accTags, tag) => (tagMap[tag] ? [...accTags, tagMap[tag]] : accTags), []);
const tags = await matchTags(rawTags);
return {
url,
@@ -120,7 +101,7 @@ function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
async function fetchLatest(site, _date) {
const res = await bhttp.get(`${site.url}/trial/categories/movies_1_d.html`);
return scrapeLatest(res.body.toString(), site);