Added Dogfart scraper. Added 'date added' property to release page.

This commit is contained in:
2019-11-04 05:47:37 +01:00
parent d734b1f0b5
commit 5745cd33d8
25 changed files with 747 additions and 102 deletions

View File

@@ -3,9 +3,9 @@
const Promise = require('bluebird');
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const knex = require('knex');
const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
async function fetchPhotos(url) {
@@ -126,7 +126,6 @@ async function scrapeScene(html, url, site) {
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
const rawTags = data.keywords.split(', ');
const siteDomain = $('meta[name="twitter:domain"]').attr('content');
const siteId = siteDomain && siteDomain.split('.')[0].toLowerCase();
const siteUrl = siteDomain && `https://www.${siteDomain}`;
@@ -136,11 +135,13 @@ async function scrapeScene(html, url, site) {
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
const rawTags = data.keywords.split(', ');
const [channelSite, tags] = await Promise.all([
site.isFallback
? knex('sites')
.where({ url: siteUrl })
.orWhere({ id: siteId })
.orWhere({ slug: siteId })
.first()
: site,
matchTags(rawTags),