traxxx/src/scrapers/ddfnetwork.js

124 lines
3.9 KiB
JavaScript

'use strict';
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
// const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.card.m-1').toArray();
return sceneElements.map((element) => {
const sceneLinkElement = $(element).find('a').first();
const title = sceneLinkElement.attr('title');
const url = `${site.url}${sceneLinkElement.attr('href')}`;
const entryId = url.split('/').slice(-1)[0];
const date = moment.utc($(element).find('small[datetime]').attr('datetime'), 'YYYY-MM-DD HH:mm:ss').toDate();
const actors = $(element).find('.card-subtitle a').map((actorIndex, actorElement) => $(actorElement).text().trim())
.toArray()
.filter(actor => actor);
const duration = parseInt($(element).find('.card-info div:nth-child(2) .card-text').text(), 10) * 60;
const poster = sceneLinkElement.find('img').attr('data-src');
return {
url,
entryId,
title,
actors,
date,
duration,
poster,
rating: null,
site,
};
});
}
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const entryId = url.split('/').slice(-1)[0];
const title = $('meta[itemprop="name"]').attr('content');
const description = $('.descr-box p').text(); // meta tags don't contain full description
const date = moment.utc($('meta[itemprop="uploadDate"]').attr('content'), 'YYYY-MM-DD').toDate();
const actors = $('.pornstar-card > a').map((actorIndex, actorElement) => $(actorElement).attr('title')).toArray();
const likes = Number($('.info-panel.likes .likes').text());
const duration = Number($('.info-panel.duration .duration').text().slice(0, -4)) * 60;
const rawTags = $('.tags-tab .tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const poster = $('#video').attr('poster');
const photos = $('.photo-slider-guest .card a').map((photoIndex, photoElement) => $(photoElement).attr('href')).toArray();
const trailer540 = $('source[res="540"]').attr('src');
const trailer720 = $('source[res="720"]').attr('src');
/*
* broken as of nov 2019
const { origin } = new URL($('.pornstar-card meta[itemprop="url"]').first().attr('content'));
const [channelSite, tags] = await Promise.all([
// don't find site if original is already specific
site.isFallback ? knex('sites').where({ url: origin }).first() : site,
matchTags(rawTags),
]);
*/
const tags = await matchTags(rawTags);
return {
// url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
url,
entryId,
title,
description,
actors,
date,
duration,
tags,
poster,
photos,
trailer: trailer540
? {
src: trailer540,
quality: 540,
}
: {
// backup
src: trailer720,
quality: 720,
},
rating: {
likes,
},
// site: channelSite || site,
site,
};
}
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
return scrapeLatest(res.body.toString(), site);
}
async function fetchScene(url, site) {
const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
return scrapeScene(res.body.toString(), url, site);
}
module.exports = {
fetchLatest,
fetchScene,
};