Major refactor, cleand up site scrape module, fixed and cleaned up release scrape module. Removed old CLI code
This commit is contained in:
@@ -3,8 +3,8 @@
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
const knex = require('knex');
|
||||
|
||||
const { fetchSites } = require('../sites');
|
||||
const { matchTags } = require('../tags');
|
||||
|
||||
function scrapeLatest(html, site) {
|
||||
@@ -71,13 +71,17 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
|
||||
|
||||
const { average: stars } = ratingRes.body;
|
||||
|
||||
const sitename = $('.shoot-logo a').attr('href').split('/')[2];
|
||||
const siteName = $('.shoot-logo a').attr('href').split('/')[2];
|
||||
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
|
||||
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
|
||||
const [channelSite, tags] = await Promise.all([
|
||||
const [[channelSite], tags] = await Promise.all([
|
||||
site.isFallback
|
||||
? knex('sites').where({ slug: sitename }).first()
|
||||
: site,
|
||||
? fetchSites({
|
||||
slug: siteSlug,
|
||||
name: siteName,
|
||||
})
|
||||
: [site],
|
||||
matchTags(rawTags),
|
||||
]);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user