Major refactor, cleand up site scrape module, fixed and cleaned up release scrape module. Removed old CLI code

This commit is contained in:
2019-11-16 03:33:36 +01:00
parent b07f88d023
commit b489c8fc33
35 changed files with 595 additions and 847 deletions

View File

@@ -3,8 +3,8 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('knex');
const { fetchSites } = require('../sites');
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
@@ -71,13 +71,17 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const { average: stars } = ratingRes.body;
const sitename = $('.shoot-logo a').attr('href').split('/')[2];
const siteName = $('.shoot-logo a').attr('href').split('/')[2];
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [channelSite, tags] = await Promise.all([
const [[channelSite], tags] = await Promise.all([
site.isFallback
? knex('sites').where({ slug: sitename }).first()
: site,
? fetchSites({
slug: siteSlug,
name: siteName,
})
: [site],
matchTags(rawTags),
]);