'use strict'; const Promise = require('bluebird'); const bhttp = require('bhttp'); const { JSDOM } = require('jsdom'); const cheerio = require('cheerio'); async function fetchPhotos(url) { const res = await bhttp.get(url); return res.body.toString(); } function scrapePhotos(html) { const $ = cheerio.load(html, { normalizeWhitespace: true }); return $('.preview .imgLink').toArray().map((linkEl) => { const url = $(linkEl).attr('href'); if (url.match('/join')) { // URL links to join page instead of full photo, extract thumbnail const src = $(linkEl).find('img').attr('src'); if (src.match('previews/')) { // resource often serves full photo at a modifier URL anyway, add as primary source const highRes = src .replace('previews/', '') .replace('_tb.jpg', '.jpg'); // keep original thumbnail as fallback in case full photo is not available return [highRes, src]; } return src; } // URL links to full photo return url; }); } async function getPhotos(albumPath, siteDomain) { const albumUrl = `https://${siteDomain}${albumPath}`; try { const html = await fetchPhotos(albumUrl); const $ = cheerio.load(html, { normalizeWhitespace: true }); const photos = scrapePhotos(html); const pages = $('.paginatorPages a').map((pageIndex, pageElement) => $(pageElement).attr('href')).toArray(); const otherPhotos = await Promise.map(pages, async (page) => { const pageUrl = `https://${siteDomain}${page}`; const pageHtml = await fetchPhotos(pageUrl); return scrapePhotos(pageHtml); }, { concurrency: 2, }); return photos.concat(otherPhotos.flat()); } catch (error) { console.error(`Failed to fetch ${siteDomain} photos from ${albumPath}: ${error.message}`); return []; } } function scrapeActorSearch(html, url, actorName) { const { document } = new JSDOM(html).window; const actorLink = document.querySelector(`a[title="${actorName}" i]`); return actorLink ? actorLink.href : null; } function scrapeProfile(html, url, actorName, siteSlug) { const { document } = new JSDOM(html).window; const avatarEl = document.querySelector('img.actorPicture'); const descriptionEl = document.querySelector('.actorBio p:not(.bioTitle)'); const hairEl = document.querySelector('.actorProfile .attribute_hair_color'); const heightEl = document.querySelector('.actorProfile .attribute_height'); const weightEl = document.querySelector('.actorProfile .attribute_weight'); const aliasEl = document.querySelector('.actorProfile .attribute_alternate_names'); const nationalityEl = document.querySelector('.actorProfile .attribute_home'); const profile = { name: actorName, }; if (avatarEl) { // larger sizes usually available, provide fallbacks const avatars = [ avatarEl.src.replace(/\d+x\d+/, '500x750'), avatarEl.src.replace(/\d+x\d+/, '240x360'), avatarEl.src.replace(/\d+x\d+/, '200x300'), avatarEl.src, ]; profile.avatar = avatars; } if (descriptionEl) profile.description = descriptionEl.textContent.trim(); if (hairEl) profile.hair = hairEl.textContent.split(':')[1].trim(); if (heightEl) profile.height = Number(heightEl.textContent.match(/\d+/)[0]); if (weightEl) profile.weight = Number(weightEl.textContent.match(/\d+/)[0]); if (aliasEl) profile.aliases = aliasEl.textContent.split(':')[1].trim().split(', '); if (nationalityEl) profile.nationality = nationalityEl.textContent.split(':')[1].trim(); profile.releases = Array.from(document.querySelectorAll('.sceneList .scene a.imgLink'), el => `https://${siteSlug}.com${el.href}`); return profile; } async function fetchProfile(actorName, siteSlug, altSearchUrl) { const actorSlug = actorName.toLowerCase().replace(/\s+/, '+'); const searchUrl = altSearchUrl ? `https://www.${siteSlug}.com/en/search/${actorSlug}/1/actor` : `https://www.${siteSlug}.com/en/search/${siteSlug}/actor/${actorSlug}`; const searchRes = await bhttp.get(searchUrl); if (searchRes.statusCode !== 200) { return null; } const actorUrl = scrapeActorSearch(searchRes.body.toString(), searchUrl, actorName); if (actorUrl) { const url = `https://${siteSlug}.com${actorUrl}`; const actorRes = await bhttp.get(url); if (actorRes.statusCode !== 200) { return null; } return scrapeProfile(actorRes.body.toString(), url, actorName, siteSlug); } return null; } async function fetchApiCredentials(referer) { const res = await bhttp.get(referer); const body = res.body.toString(); const apiLine = body.split('\n').find(bodyLine => bodyLine.match('apiKey')); const apiSerial = apiLine.slice(apiLine.indexOf('{'), apiLine.indexOf('};') + 1); const apiData = JSON.parse(apiSerial); const { applicationID: appId, apiKey } = apiData.api.algolia; const userAgent = 'Algolia for vanilla JavaScript (lite) 3.27.0;instantsearch.js 2.7.4;JS Helper 2.26.0'; const apiUrl = `https://${appId.toLowerCase()}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${userAgent}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`; return { appId, apiKey, userAgent, apiUrl, }; } async function fetchLatest(site, page = 1, upcoming = false) { const referer = `${site.url}/en/videos`; const { apiUrl } = await fetchApiCredentials(referer); console.log(referer); const res = await bhttp.post(apiUrl, { requests: [ { indexName: 'all_scenes', params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]`, }, ], }, { headers: { Referer: referer, }, encodeJSON: true, }); console.log(res.body.results); // return scrape(res.body.results[0].hits, site); } module.exports = { getPhotos, fetchProfile, scrapeProfile, fetchLatest, };