Added Dogfart scraper. Added 'date added' property to release page.

This commit is contained in:
2019-11-04 05:47:37 +01:00
parent d734b1f0b5
commit 5745cd33d8
25 changed files with 747 additions and 102 deletions

View File

@@ -3,8 +3,8 @@
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const knex = require('knex');
const knex = require('../knex');
const { matchTags } = require('../tags');
function scrapeLatest(html, site) {
@@ -75,7 +75,9 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
const rawTags = $('.tag-list > a[href*="/tag"]').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [channelSite, tags] = await Promise.all([
knex('sites').where({ slug: sitename }).first(),
site.isFallback
? knex('sites').where({ slug: sitename }).first()
: site,
matchTags(rawTags),
]);