Added Dogfart scraper. Added 'date added' property to release page.

src/scrapers/dogfart.js

@@ -0,0 +1,170 @@
+'use strict';
+
+/* eslint-disable newline-per-chained-call */
+const Promise = require('bluebird');
+const bhttp = require('bhttp');
+const { JSDOM } = require('jsdom');
+const moment = require('moment');
+const knex = require('knex');
+
+const { matchTags } = require('../tags');
+
+async function getPhoto(url) {
+    const res = await bhttp.get(url);
+    const html = res.body.toString();
+    const { document } = new JSDOM(html).window;
+
+    const photoUrl = document.querySelector('.scenes-module img').src;
+
+    return photoUrl;
+}
+
+async function getPhotos(albumUrl, site, siteUrl) {
+    const res = await bhttp.get(albumUrl);
+    const html = res.body.toString();
+    const { document } = new JSDOM(html).window;
+
+    const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
+    const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);
+
+    // dogfart has massive albums, pick 20 or specified photos: first, last and evenly inbetween
+    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
+    const photoIndexes = [1]
+        .concat(Array.from({ length: photoLimit - 2 }, (value, index) => Math.floor((index + 1) * (lastPhotoIndex / (photoLimit - 2)))))
+        .concat(lastPhotoIndex);
+
+    if (photoLimit > 25) {
+        console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
+    }
+
+    const photoUrls = await Promise.map(photoIndexes, async (index) => {
+        const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`;
+
+        return getPhoto(pageUrl);
+    }, {
+        concurrency: 5,
+    });
+
+    return photoUrls;
+}
+
+function scrapeLatest(html, site) {
+    const { document } = new JSDOM(html).window;
+    const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));
+
+    return sceneElements.reduce((acc, element) => {
+        const siteUrl = element.querySelector('.help-block').textContent;
+
+        if (siteUrl.toLowerCase() !== new URL(site.url).host) {
+            // different dogfart site
+            return acc;
+        }
+
+        const sceneLinkElement = element.querySelector('.thumbnail');
+        const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
+        const { pathname } = new URL(url);
+        const entryId = `${site.slug}_${pathname.split('/')[4]}`;
+
+        const title = element.querySelector('.scene-title').textContent;
+        const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
+
+        const poster = `https:${element.querySelector('img').src}`;
+        const trailer = sceneLinkElement.dataset.preview_clip_url;
+
+        return [
+            ...acc,
+            {
+                url,
+                entryId,
+                title,
+                actors,
+                poster,
+                trailer: {
+                    src: trailer,
+                },
+                site,
+            },
+        ];
+    }, []);
+}
+
+async function scrapeScene(html, url, site) {
+    const { document } = new JSDOM(html).window;
+
+    const title = document.querySelector('.description-title').textContent;
+    const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
+    const metaDescription = document.querySelector('meta[itemprop="description"]').content;
+    const description = metaDescription
+        ? metaDescription.content
+        : document.querySelector('.description')
+            .textContent
+            .replace(/[ \t\n]{2,}/g, ' ')
+            .replace('...read more', '')
+            .trim();
+
+    const siteSlug = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
+    const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
+    const duration = moment
+        .duration(document
+            .querySelectorAll('.extra-info p')[1]
+            .textContent
+            .match(/\d+:\d+$/)[0])
+        .asSeconds();
+
+    const trailerElement = document.querySelector('.html5-video');
+    const poster = `https:${trailerElement.dataset.poster}`;
+    const { trailer } = trailerElement.dataset;
+
+    const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
+    const { origin, pathname } = new URL(url);
+    const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
+
+    const stars = Number(document.querySelector('span[itemprop="average"]').textContent) / 2;
+    const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
+
+    const [channelSite, tags] = await Promise.all([
+        site.isFallback
+            ? knex('sites')
+                .where({ slug: siteSlug })
+                .orWhere({ url: `https://${siteSlug}.com` })
+                .first()
+            : site,
+        matchTags(rawTags),
+    ]);
+
+    return {
+        url,
+        title,
+        description,
+        actors,
+        date,
+        duration,
+        poster,
+        photos,
+        trailer: {
+            src: trailer,
+        },
+        tags,
+        rating: {
+            stars,
+        },
+        site: channelSite || site,
+    };
+}
+
+async function fetchLatest(site, page = 1) {
+    const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);
+
+    return scrapeLatest(res.body.toString(), site);
+}
+
+async function fetchScene(url, site) {
+    const res = await bhttp.get(url);
+
+    return scrapeScene(res.body.toString(), url, site);
+}
+
+module.exports = {
+    fetchLatest,
+    fetchScene,
+};