Added Dogfart scraper. Added 'date added' property to release page.

This commit is contained in:
2019-11-04 05:47:37 +01:00
parent d734b1f0b5
commit 5745cd33d8
25 changed files with 747 additions and 102 deletions

View File

@@ -13,6 +13,8 @@ function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.card.card--release').toArray();
console.log(sceneElements);
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('.card-info__title a');
const title = sceneLinkElement.attr('title');
@@ -22,6 +24,8 @@ function scrapeLatest(html, site) {
const date = moment.utc($(element).find('.card-info__meta-date').text(), 'MMMM DD, YYYY').toDate();
const actors = $(element).find('.card-info__cast a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
console.log(date, actors, title);
return {
url,
entryId,
@@ -54,6 +58,8 @@ async function scrapeScene(data, url, site) {
const { likes, dislikes } = data.stats;
const duration = data.videos.mediabook.length;
console.log(data);
const rawTags = data.tags.map(tag => tag.name);
const tags = await matchTags(rawTags);