traxxx/src/scrapers/dogfart.js

'use strict';

/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const knex = require('knex');

const { matchTags } = require('../tags');
const pluckPhotos = require('../utils/pluck-photos');

async function getPhoto(url) {
    const res = await bhttp.get(url);
    const html = res.body.toString();
    const { document } = new JSDOM(html).window;

    const photoUrl = document.querySelector('.scenes-module img').src;

    return photoUrl;
}

async function getPhotos(albumUrl, site, siteUrl) {
    const res = await bhttp.get(albumUrl);
    const html = res.body.toString();
    const { document } = new JSDOM(html).window;

    const lastPhotoPage = Array.from(document.querySelectorAll('.preview-image-container a')).slice(-1)[0].href;
    const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10);

    // dogfart has massive albums, pick 25 or specified number of photos: first, last and evenly inbetween
    const photoLimit = (site.network.parameters && site.network.parameters.photoLimit) || 25;
    const photoIndexes = pluckPhotos(lastPhotoIndex, photoLimit);

    if (photoLimit > 25) {
        console.log(`${site.name}: Scraping ${photoLimit} album photos from ${siteUrl}, this may take some time...`);
    }

    const photoUrls = await Promise.map(photoIndexes, async (index) => {
        const pageUrl = `https://blacksonblondes.com${lastPhotoPage.replace(/\d+.jpg/, `${index.toString().padStart(3, '0')}.jpg`)}`;

        return getPhoto(pageUrl);
    }, {
        concurrency: 5,
    });

    return photoUrls;
}

function scrapeLatest(html, site) {
    const { document } = new JSDOM(html).window;
    const sceneElements = Array.from(document.querySelectorAll('.recent-updates'));

    return sceneElements.reduce((acc, element) => {
        const siteUrl = element.querySelector('.help-block').textContent;

        if (siteUrl.toLowerCase() !== new URL(site.url).host) {
            // different dogfart site
            return acc;
        }

        const sceneLinkElement = element.querySelector('.thumbnail');
        const url = `https://dogfartnetwork.com${sceneLinkElement.href}`;
        const { pathname } = new URL(url);
        const entryId = `${site.slug}_${pathname.split('/')[4]}`;

        const title = element.querySelector('.scene-title').textContent;
        const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());

        const poster = `https:${element.querySelector('img').src}`;
        const trailer = sceneLinkElement.dataset.preview_clip_url;

        return [
            ...acc,
            {
                url,
                entryId,
                title,
                actors,
                poster,
                trailer: {
                    src: trailer,
                },
                site,
            },
        ];
    }, []);
}

async function scrapeScene(html, url, site) {
    const { document } = new JSDOM(html).window;

    const title = document.querySelector('.description-title').textContent;
    const actors = Array.from(document.querySelectorAll('.more-scenes a')).map(({ textContent }) => textContent);
    const metaDescription = document.querySelector('meta[itemprop="description"]').content;
    const description = metaDescription
        ? metaDescription.content
        : document.querySelector('.description')
            .textContent
            .replace(/[ \t\n]{2,}/g, ' ')
            .replace('...read more', '')
            .trim();

    const siteSlug = document.querySelector('.site-name').textContent.split('.')[0].toLowerCase();
    const date = new Date(document.querySelector('meta[itemprop="uploadDate"]').content);
    const duration = moment
        .duration(document
            .querySelectorAll('.extra-info p')[1]
            .textContent
            .match(/\d+:\d+$/)[0])
        .asSeconds();

    const trailerElement = document.querySelector('.html5-video');
    const poster = `https:${trailerElement.dataset.poster}`;
    const { trailer } = trailerElement.dataset;

    const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
    const { origin, pathname } = new URL(url);
    const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);

    const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
    const rawTags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);

    const [channelSite, tags] = await Promise.all([
        site.isFallback
            ? knex('sites')
                .where({ slug: siteSlug })
                .orWhere({ url: `https://${siteSlug}.com` })
                .first()
            : site,
        matchTags(rawTags),
    ]);

    return {
        url: `${origin}${pathname}`,
        title,
        description,
        actors,
        date,
        duration,
        poster,
        photos,
        trailer: {
            src: trailer,
        },
        tags,
        rating: {
            stars,
        },
        site: channelSite || site,
    };
}

async function fetchLatest(site, page = 1) {
    const res = await bhttp.get(`https://dogfartnetwork.com/tour/scenes/?p=${page}`);

    return scrapeLatest(res.body.toString(), site);
}

async function fetchScene(url, site) {
    const res = await bhttp.get(url);

    return scrapeScene(res.body.toString(), url, site);
}

module.exports = {
    fetchLatest,
    fetchScene,
};