Added Dogfart scraper. Added 'date added' property to release page.
This commit is contained in:
@@ -3,9 +3,9 @@
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
const cheerio = require('cheerio');
|
||||
const knex = require('knex');
|
||||
const moment = require('moment');
|
||||
|
||||
const knex = require('../knex');
|
||||
const { matchTags } = require('../tags');
|
||||
|
||||
async function fetchPhotos(url) {
|
||||
@@ -126,7 +126,6 @@ async function scrapeScene(html, url, site) {
|
||||
|
||||
const duration = moment.duration(data.duration.slice(2).split(':')).asSeconds();
|
||||
|
||||
const rawTags = data.keywords.split(', ');
|
||||
const siteDomain = $('meta[name="twitter:domain"]').attr('content');
|
||||
const siteId = siteDomain && siteDomain.split('.')[0].toLowerCase();
|
||||
const siteUrl = siteDomain && `https://www.${siteDomain}`;
|
||||
@@ -136,11 +135,13 @@ async function scrapeScene(html, url, site) {
|
||||
|
||||
const photos = await getPhotos($('.picturesItem a').attr('href'), siteDomain);
|
||||
|
||||
const rawTags = data.keywords.split(', ');
|
||||
|
||||
const [channelSite, tags] = await Promise.all([
|
||||
site.isFallback
|
||||
? knex('sites')
|
||||
.where({ url: siteUrl })
|
||||
.orWhere({ id: siteId })
|
||||
.orWhere({ slug: siteId })
|
||||
.first()
|
||||
: site,
|
||||
matchTags(rawTags),
|
||||
|
||||
Reference in New Issue
Block a user