'use strict'; const util = require('util'); const Promise = require('bluebird'); const unprint = require('unprint'); const argv = require('../argv'); const qu = require('../utils/qu'); const { heightToCm } = require('../utils/convert'); const slugify = require('../utils/slugify'); function getEntryId(html) { const entryId = html.match(/showtagform\((\d+)\)/); if (entryId) { return entryId[1]; } const setIdIndex = html.indexOf('setid:"'); if (setIdIndex) { return html.slice(setIdIndex, html.indexOf(',', setIdIndex)).match(/\d+/)?.[0]; } return null; } function scrapeAll(scenes, site, entryIdFromTitle) { return scenes.map(({ element, query }) => { const release = {}; const title = query.content('.content_img div, .dvd_info > a, a.update_title, a[title] + a[title], .overlay-text') || query.content('a[title*=" "]'); release.title = title?.slice(0, title.match(/starring:/i)?.index || Infinity).trim(); release.url = query.url('.content_img a, .dvd_info > a, a.update_title, a[title]'); release.date = query.date('.update_date', 'MM/DD/YYYY'); release.entryId = (entryIdFromTitle && slugify(release.title)) || element.dataset.setid || query.element('.rating_box')?.dataset.id || query.attribute('a img', 'id')?.match(/set-target-(\d+)/)?.[1]; release.actors = query.all('.content_img .update_models a, .update_models a').map((actorEl) => ({ name: unprint.query.content(actorEl), url: unprint.query.url(actorEl, null), })); const dvdPhotos = query.imgs('.dvd_preview_thumb'); const photoCount = Number(query.attribute('a img.thumbs', 'cnt')) || 1; [release.poster, ...release.photos] = dvdPhotos.length ? dvdPhotos : Array.from({ length: photoCount }).map((value, index) => { const src = query.img('a img.thumbs', { attribute: `src${index}_1x` }) || query.img('a img.thumbs', { attribute: `src${index}` }) || query.img('a img.thumbs'); const prefixedSrc = qu.prefixUrl(src, site.url); if (src) { return Array.from(new Set([ prefixedSrc.replace(/.jpg$/, '-full.jpg'), prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'), prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'), prefixedSrc, ])).map((source) => ({ src: source, referer: site.url, verifyType: 'image', })); } return null; }).filter(Boolean); const teaserScript = query.html('script'); if (teaserScript) { release.teaser = teaserScript.slice(teaserScript.indexOf('http'), teaserScript.indexOf('.mp4') + 4); } return release; }); } function scrapeUpcoming(scenes, channel) { return scenes.map(({ query, html }) => { const release = {}; release.title = query.text('.overlay-text', { join: false })?.[0]; release.date = query.date('.overlay-text', 'MM/DD/YYYY'); release.actors = query.all('.update_models a').map((actorEl) => ({ name: unprint.query.content(actorEl), url: unprint.query.url(actorEl, null), })); release.poster = query.img('img') || query.img('img', { attribute: 'src0_1x' }); release.teaser = html.match(/src=['"](https:\/\/.*\.mp4)['"]/)?.[1]; release.entryId = channel.parameters?.entryIdFromTitle ? slugify(release.title) : getEntryId(html); return release; }); } function extractLegacyTrailer(html, context) { const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line)); if (trailerLines.length) { return trailerLines.map((trailerLine) => { // const src = trailerLine.match(/path:"([\w-:/.&=?%]+)"/)?.[1]; const src = trailerLine.match(/path:"(.+)"/)?.[1]; const quality = trailerLine.match(/movie_height:'(\d+)/)?.[1]; return src && { src: /^http/.test(src) ? src : `${context.entity.url}${src}`, quality: quality && Number(quality.replace('558', '540')), }; }).filter(Boolean); } return null; } const qualities = [ 'photos', '1600watermarked', '1280watermarked', '1024watermarked', 'thumbs', ]; function getPhotos(query, release, context) { // https://thumbs.julesjordan.com/members/content//upload/dl03/julesjordan/whitney_wright_dredd/1024watermarked/whitney_wright_julesjordan.com-20.jpg // https://thumbs.julesjordan.com/members/content//upload/dl03/julesjordan/bambi_barton_manuel_ferrara/1024watermarked/bambi_barton_julesjordan_com-13.jpg if (!release.actors?.length > 0) { return null; } const photoCount = query.number('//div[contains(@class, "title-heading-content")][contains(text(), "Photos")]'); if (photoCount) { // slug actor order is not always the same as actor list order, prefer trailer slug if available const path = query.dataset('.movieformat_button', 'src')?.match(/:(.*)_trailer/)?.[1] || release.actors.map((actor) => slugify(actor.name || actor, '_')).join('_'); const derivedActorSlug = path.replace(`_${release.actors.slice(1).map(({ name }) => slugify(name, '_'))}`, ''); const actorSlug = derivedActorSlug === path // no replacement took place, so the slug is likely invalid ? slugify(release.actors[0].name || release.actors[0], '_') : derivedActorSlug; return Array.from({ length: photoCount }, (value, index) => qualities .flatMap((quality) => [ `https://thumbs.${context.entity.slug}.com/trial/content//upload/dl03/${context.entity.slug}/${path}/${quality}/${actorSlug}_${context.entity.slug}_com-${index + 1}.jpg`, `https://thumbs.${context.entity.slug}.com/trial/content//upload/dl03/${context.entity.slug}/${path}/${quality}/${actorSlug}_${context.entity.slug}.com-${index + 1}.jpg`, // .com instead of _com ]).map((src) => ({ src, attempts: 1 }))); } return null; } async function scrapeScene({ html, query }, context) { const release = {}; release.title = query.content('.title_bar_hilite, .movie_title'); release.description = query.content('.update_description') || query.text('//div[./span[contains(text(), "Description")]]'); release.entryId = context.entity.parameters?.entryIdFromTitle ? slugify(release.title) : getEntryId(html); release.date = query.date(['.update_date', '//div[./span[contains(text(), "Date")]]'], 'MM/DD/YYYY'); release.actors = query.all('.backgroundcolor_info > .update_models a, .item .update_models a, .player-scene-description .update_models a').map((actorEl) => ({ name: unprint.query.content(actorEl), url: unprint.query.url(actorEl, null), })); release.tags = query.contents('.update_tags a, .player-scene-description a[href*="/categories"]'); release.director = release.tags?.find((tag) => ['mike john', 'van styles'].includes(tag?.trim().toLowerCase())); const posterPath = query.poster('#video-player') || html.match(/useimage = "(.*)"/)?.[1]; if (posterPath) { const poster = /^http/.test(posterPath) ? posterPath : `${context.entity.url}${posterPath}`; if (poster) { release.poster = { src: poster, referer: context.entity.url, }; } } if (query.exists('source[data-bitrate="trailer"]')) { release.trailer = [ query.video('source[data-bitrate="trailer_1080" i]'), query.video('source[data-bitrate="trailer_720" i]'), query.video('source[data-bitrate="trailer" i]'), // also seems to be 720p query.video('source[data-bitrate="trailer_mobile" i]'), // also seems to be 720p ]; } else if (context.include.trailers && context.entity.slug !== 'manuelferrara') { release.trailer = extractLegacyTrailer(html, context); } // release.photos = async () => await getPhotos(release.entryId, context.entity); // probably no longer works on any site if (argv.jjFullPhotos) { release.photos = getPhotos(query, release, context); } else { // base release photos are usually better, but deep photos have additional thumbs // the filenames are not chronological, so sorting after appending only worsens the mix release.photos = [ ...context.baseRelease?.photos?.map((sources) => sources.at(-1).src) || [], ...query.imgs('#images img'), ].map((source) => Array.from(new Set([ source.replace(/.jpg$/, '-full.jpg'), source.replace(/-1x.jpg$/, '-4x.jpg'), source.replace(/-1x.jpg$/, '-2x.jpg'), source, ])).map((fallbackSource) => ({ src: fallbackSource, referer: context.entity.url, verifyType: 'image', }))); } if (query.exists('.update_dvds a')) { release.movie = { url: query.url('.update_dvds a'), title: query.cnt('.update_dvds a'), }; release.movie.entryId = new URL(release.movie.url).pathname.split('/').slice(-1)[0]?.replace('.html', ''); } release.stars = query.number('.avg_rating'); return release; } function scrapeMovie({ el, query }, url, site) { const movie = { url, site }; movie.entryId = new URL(url).pathname.split('/').slice(-1)[0]?.replace('.html', ''); movie.title = query.cnt('.title_bar span'); movie.covers = query.urls('#dvd-cover-flip > a'); movie.channel = slugify(query.q('.update_date a', true), ''); // movie.releases = Array.from(document.querySelectorAll('.cell.dvd_info > a'), el => el.href); const sceneQus = qu.initAll(el, '.dvd_details'); const scenes = scrapeAll(sceneQus, site); const curatedScenes = scenes ?.map((scene) => ({ ...scene, movie })) .sort((sceneA, sceneB) => sceneA.date - sceneB.date); movie.date = curatedScenes?.[0]?.date; return { ...movie, ...(curatedScenes && { scenes: curatedScenes }), }; } function scrapeProfile({ query }, url, name, entity) { const profile = { url }; profile.description = query.content('//comment()[contains(., " Bio Extra Field ")]/following-sibling::span'); // the spaces are important to avoid selecting a similar comment profile.height = heightToCm(query.content('//span[contains(text(), "Height")]/following-sibling::span')); profile.measurements = query.content('//span[contains(text(), "Measurements")]/following-sibling::span'); const age = query.content('//span[contains(text(), "Age")]/following-sibling::span')?.trim(); if (age && /\w+ \d+, \d{4}/.test(age)) { profile.dateOfBirth = unprint.extractDate(age, 'MMMM D, YYYY'); } else { profile.age = Number(age) || null; } profile.avatar = [ query.img('.model_bio_pic img, .model_bio_thumb', { attribute: 'src0_3x' }), query.img('.model_bio_pic img, .model_bio_thumb', { attribute: 'src0_2x' }), query.img('.model_bio_pic img, .model_bio_thumb', { attribute: 'src0_1x' }), query.img('.model_bio_pic img, .model_bio_thumb', { attribute: 'src0' }), query.img('.model_bio_pic img, .model_bio_thumb', { attribute: 'src' }), ].filter(Boolean); profile.scenes = scrapeAll(unprint.initAll(query.all('.grid-item')), entity, true); return profile; } async function fetchLatest(site, page = 1, include, preData, entryIdFromTitle = false) { const url = site.parameters?.latest ? util.format(site.parameters.latest, page) : `${site.url}/trial/categories/movies_${page}_d.html`; // const res = await http.get(url); const res = await unprint.get(url, { selectAll: '.update_details, .grid-item' }); if (res.ok) { return scrapeAll(res.context, site, typeof site.parameters?.entryIdFromTitle === 'boolean' ? site.parameters.entryIdFromTitle : entryIdFromTitle); } return res.status; } async function fetchUpcoming(site) { if (site.parameters?.upcoming === false) return null; const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`; const res = await unprint.get(url, { selectAll: '//img[contains(@alt, "Coming Soon")]/parent::div' }); if (res.ok) { return scrapeUpcoming(res.context, site); } return res.status; } async function fetchMovie(url, site) { const res = await qu.get(url); return res.ok ? scrapeMovie(res.item, url, site) : res.status; } async function fetchProfile({ name: actorName, url }, entity) { const actorSlugA = slugify(actorName, ''); const actorSlugB = slugify(actorName, '-'); const urls = [ url, `${entity.parameters?.profile || `${entity.url}/trial/models`}/${actorSlugA}.html`, `${entity.parameters?.profile || `${entity.url}/trial/models`}/${actorSlugB}.html`, ]; return urls.reduce(async (chain, profileUrl) => { const profile = await chain; if (profile) { return profile; } if (!profileUrl) { return null; } const res = await unprint.get(profileUrl); if (res.ok) { return scrapeProfile(res.context, profileUrl, actorName, entity); } return null; }, Promise.resolve()); } module.exports = { fetchLatest, fetchMovie, fetchProfile, fetchUpcoming, scrapeScene: { scraper: scrapeScene, unprint: true, }, };