'use strict'; const slugify = require('../utils/slugify'); const qu = require('../utils/qu'); async function getPhotos(albumUrl, channel) { const res = await qu.get(albumUrl); if (!res.ok) { return []; } const lastPhotoPage = res.item.query.urls('.pics-container .preview-image-container a').at(-1); const lastPhotoIndex = parseInt(lastPhotoPage.match(/\d+.jpg/)[0], 10); const photoUrls = Array.from({ length: lastPhotoIndex }, (value, index) => { const pageUrl = `${channel.url}${lastPhotoPage.replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`; const networkPageUrl = `https://dogfartnetwork.com${lastPhotoPage.replace('tourx', 'tour').replace(/\d+.jpg/, `${(index + 1).toString().padStart(3, '0')}.jpg`)}`; const extract = ({ query }) => query.img('.scenes-module img'); return [ { url: pageUrl, extract, }, { url: networkPageUrl, extract, }, ]; }); return photoUrls; } function scrapeLatest(scenes, site, filter = true) { return scenes.reduce((acc, { query }) => { const release = {}; const siteUrl = query.cnt('.recent-details-title .help-block, .model-details-title .site-name'); release.url = query.url('.thumbnail, .preview-image-container > a', 'href', { origin: site.url }); release.entryId = `${site.slug}_${new URL(release.url).pathname.split('/')[4]}`; release.title = query.cnt('.scene-title'); // release.actors = release.title.split(/[,&]|\band\b/).map((actor) => actor.replace(/BTS/i, '').trim()); // the titles don't always list the actors, e.g. BarbCummings.com // release.poster = `https:${element.querySelector('img').src}`; release.poster = query.img(); release.teaser = query.video('.thumbnail, .preview-thumbnail', 'data-preview_clip_url'); release.channel = siteUrl?.match(/(.*).com/)?.[1].toLowerCase(); if (filter && siteUrl && `www.${siteUrl.toLowerCase()}` !== new URL(site.url).host) { // different dogfart site return { ...acc, unextracted: [...acc.unextracted, release] }; } return { ...acc, scenes: [...acc.scenes, release] }; }, { scenes: [], unextracted: [], }); } async function scrapeScene({ query }, url, channel, baseScene, parameters) { const release = {}; const { origin, pathname } = new URL(url); release.channel = channel.type === 'channel' ? channel.slug : query.cnt('.site-name').split('.')[0].toLowerCase(); release.entryId = `${release.channel}_${pathname.split('/').slice(-2)[0]}`; release.title = query.cnt('.description-title') || query.text('.scene-title'); release.actors = query.all('.more-scenes a, .starring-list a').map((actorEl) => ({ name: query.cnt(actorEl), url: query.url(actorEl, null, 'href', { origin: channel.url }), })); release.description = query.meta('meta[itemprop="description"]') || query.cnt('.description, [itemprop="description"]')?.replace(/[ \t\n]{2,}/g, ' ').replace('...read more', '').trim(); release.date = query.date('meta[itemprop="uploadDate"]', null, null, 'content'); release.duration = query.duration('.extra-info p:nth-child(2), .run-time-container'); release.tags = query.exists('.scene-details .categories a') ? query.cnts('.scene-details .categories a') : query.text('.categories')?.split(/,\s+/); const trailer = query.video('.html5-video', 'data-trailer'); const lastPhotosUrl = query.urls('.pagination a').at(-1); release.poster = query.poster('.html5-video', 'data-poster') || query.img('.trailer-image'); if (trailer && !trailer?.includes('join')) { release.trailer = trailer; } if (lastPhotosUrl && parameters.includePhotos) { release.photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, channel, url); } release.stars = Number(((query.number('span[itemprop="average"], span[itemprop="ratingValue"]') || query.number('canvas[data-score]', null, 'data-score')) / 2).toFixed(2)); return release; } async function fetchLatest(channel, page = 1, { parameters }) { const res = await qu.getAll(parameters.latest ? `${parameters.latest}/?p=${page}` : `${channel.url}/tour/scenes/?p=${page}`, '.recent-updates, .preview-image-container'); if (res.ok) { return scrapeLatest(res.items, channel); } return res.status; } async function fetchProfile(baseActor, entity) { const slug = slugify(baseActor.name, '+'); const url = `https://www.dogfartnetwork.com/tour/girls/${slug}/`; const res = await qu.getAll(url, '.recent-updates'); if (res.ok) { const { scenes } = scrapeLatest(res.items, entity, false); // no bio available return { scenes }; } return res.status; } module.exports = { fetchLatest, fetchProfile, scrapeScene, };