diff --git a/migrations/20240125011700_manticore.js b/migrations/20240125011700_manticore.js index a997484e..d9fae1ee 100644 --- a/migrations/20240125011700_manticore.js +++ b/migrations/20240125011700_manticore.js @@ -83,39 +83,41 @@ const actorsFields = ` scenes int `; -exports.up = async (knex) => { +exports.up = async () => { try { - await utilsApi.sql(`create table scenes (${scenesFields})`); + await utilsApi.sql(`create table if not exists scenes (${scenesFields})`); - await utilsApi.sql(`create table scenes_stashed ( + await utilsApi.sql(`create table if not exists scenes_stashed ( scene_id int, stash_id int, user_id int, created_at timestamp )`); - await utilsApi.sql(`create table movies (${moviesFields})`); + await utilsApi.sql(`create table if not exists movies (${moviesFields})`); - await utilsApi.sql(`create table movies_stashed ( + await utilsApi.sql(`create table if not exists movies_stashed ( movie_id int, stash_id int, user_id int, created_at timestamp )`); - await utilsApi.sql(`create table actors (${actorsFields}) min_prefix_len='3'`); + await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`); - await utilsApi.sql(`create table actors_stashed ( + await utilsApi.sql(`create table if not exists actors_stashed ( actor_id int, stash_id int, user_id int, created_at timestamp )`); + /* await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id')); await knex.schema.alterTable('stashes_movies', (table) => table.increments('id')); await knex.schema.alterTable('stashes_actors', (table) => table.increments('id')); await knex.schema.alterTable('stashes_series', (table) => table.increments('id')); + */ } catch (error) { console.log(error); } diff --git a/src/scrapers/private.js b/src/scrapers/private.js index 6ac5988a..d4e885b4 100755 --- a/src/scrapers/private.js +++ b/src/scrapers/private.js @@ -1,138 +1,150 @@ 'use strict'; -/* eslint-disable newline-per-chained-call */ -const cheerio = require('cheerio'); -const moment = require('moment'); +const unprint = require('unprint'); -const { get, geta } = require('../utils/q'); const slugify = require('../utils/slugify'); -const http = require('../utils/http'); + +const headers = { + 'Accept-Language': 'en-US,en', // will translate to server language if not specified +}; + +function scrapeAll(scenes) { + return scenes.map(({ query }) => { + const release = {}; + + release.url = query.url('h3 a'); + release.entryId = new URL(release.url).pathname.split('/').at(-1); + + release.title = query.content('h3 a') + || query.attribute('a img', 'alt')?.split(/:(.*)/)[1]?.trim() + || query.attribute('a img', 'alt'); + + release.date = query.date('.scene-date', ['MM/DD/YYYY', 'YYYY-MM-DD']); + + release.actors = query.all('.scene-models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); + + release.poster = query.img('a img'); + release.teaser = query.video('.mini_video_player source'); + + console.log(release); + + return release; + }); +} + +async function fetchLatest(site, page = 1) { + const { hostname } = new URL(site.url); + + if (hostname.match('private.com')) { + const res = await unprint.get(`${site.url}/${page}/`, { selectAll: '.content-wrapper .scene', headers }); + + return scrapeAll(res.context, site); + } + + const res = await unprint.get(`${site.url}/scenes/${page}/`, { selectAll: '.content-wrapper .scene', headers }); + + return scrapeAll(res.context, site); +} async function getPhotos(entryId, site) { const { hostname } = new URL(site.url); - const res = await http.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`); - const html = res.body.toString(); + const res = await unprint.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`); - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const photos = $('a.fakethumb').map((photoIndex, photoElement) => $(photoElement).attr('data-src') || $(photoElement).attr('href')).toArray(); - - return photos; -} - -function scrapeLatest(html, site) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const sceneElements = $('.content-wrapper .scene').toArray(); - - return sceneElements.map((element) => { - const sceneLinkElement = $(element).find('h3 a'); - const thumbnailElement = $(element).find('a img'); - - const url = sceneLinkElement.attr('href'); - // const title = sceneLinkElement.text(); - const entryId = url.split('/').slice(-1)[0]; - - const titleText = thumbnailElement.attr('alt'); - const title = titleText.slice(titleText.indexOf(':') + 1).trim(); - - const date = moment.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate(); - - const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - const likes = Number($(element).find('.scene-votes').text()); - - const photoCount = Number(thumbnailElement.attr('thumbs_num')); - const poster = thumbnailElement.attr('src'); - const photos = Array.from({ length: photoCount }, (val, index) => thumbnailElement.attr(`src${index + 1}`)); - - const scene = { - url, - entryId, - title, - actors, - date, - poster, - photos, - rating: { - likes, - }, - site, - }; - - return scene; - }); -} - -async function scrapeScene(html, url, site) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const release = { url }; - - [release.entryId] = url.split('/').slice(-1); - release.title = $('.video-wrapper meta[itemprop="name"]').attr('content'); - release.description = $('.video-wrapper meta[itemprop="description"]').attr('content'); - - release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate(); - release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - - const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content'); - - if (timestamp) { - const [minutes, seconds] = timestamp.match(/\d+/g); - release.duration = Number(minutes) * 60 + Number(seconds); + if (res.ok) { + return res.context.query.urls('#lightgallery a.item'); } - release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); - release.likes = Number($('.content-desc #social-actions #likes').text()); + return []; +} - const posterScript = $('script:contains(poster)').html(); - const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4); - release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src'); +async function scrapeScene({ query }, { url, entity }) { + const release = {}; - const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'); + release.entryId = new URL(url).pathname.split('/').at(-1); + release.channel = slugify(query.content('.title-site'), ''); - if (trailer) release.trailer = { src: trailer }; + release.title = query.content(['.title-zone h1', '.content-desc h1']); + release.description = query.content('meta[itemprop="description"]', { attribute: 'content' }) || query.content('#description-section'); - release.photos = await getPhotos(release.entryId, site); - release.movie = $('a[data-track="FULL MOVIE"]').attr('href'); + release.date = query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' }); + release.duration = query.duration('meta[itemprop="duration"]', { attribute: 'content' }); - const siteElement = $('.content-wrapper .logos-sites a'); - if (siteElement) release.channel = slugify(siteElement.text(), ''); + release.actors = query.all('.title-zone .tag-models a, .scene-models-list a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); + + release.tags = query.contents('.title-zone .tag-tags, .scene-tags a'); + + release.poster = query.img('meta[itemprop="thumbnailUrl"]', { attribute: 'content' }) || query.img('#trailer_player_finished img'); + release.photos = await getPhotos(release.entryId, entity); + release.trailer = query.url('meta[itemprop="contentURL"]', { attribute: 'content' }) || query.video('#trailer_player source'); + + const movieUrl = query.url('.content-desc a[href*="movie/"]'); + + if (movieUrl) { + release.movie = { + url: movieUrl, + entryId: new URL(movieUrl).pathname.match(/movie\/(\d+)/)?.[1], + }; + } return release; } -function scrapeProfile({ html, q, qa, qtx }) { +async function scrapeMovie({ query }, { url }) { + const release = {}; + + release.entryId = new URL(url).pathname.match(/movie\/(\d+)/)[1]; + + release.title = query.content('.dvds-wrapper h1'); + release.description = query.content('meta[itemprop="description"]', { attribute: 'content' }) || query.content('p.sinopsys'); + + release.date = query.date('.date-duration [itemprop="datePublished"]', 'YYYY-MM-DD') + || query.date('.date-duration em + span', 'DD/MM/YYYY') + || query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' }); // upload date does not match release date + + release.duration = query.number('.date-duration', { match: /(\d+) min/, matchIndex: 1 }) * 60 || null; + + release.actors = query.all('.dvd-performers a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); + + release.poster = query.img('meta[itemprop="thumbnailUrl"]', { attribute: 'content' }) || query.img('#trailer_player_finished img'); + release.covers = query.imgs('.dvds-picture img'); + + release.trailer = query.url('meta[itemprop="contentURL"]', { attribute: 'content' }) || query.video('#trailer_player source'); + + release.scenes = scrapeAll(unprint.initAll(query.all('.scenes .card'))); + + return release; +} + +function scrapeProfile({ query }) { const profile = {}; - const bio = qa('.model-facts li:not(.model-facts-long)', true).reduce((acc, fact) => { - const [key, value] = fact.split(':'); - const trimmedValue = value.trim(); + const bioKeys = query.contents('.bio-pornstar .model-facts:last-child em'); + const bioValues = query.texts('.bio-pornstar .model-facts:last-child li'); - if (trimmedValue.length === 0 || trimmedValue === '-') return acc; - return { ...acc, [slugify(key, '_')]: trimmedValue }; - }, {}); + const bio = Object.fromEntries(bioKeys.map((key, index) => [slugify(key, '_'), bioValues[index]]).filter(([, value]) => value !== '-')); - const description = q('.model-facts-long', true); - if (description) profile.description = description; + profile.description = query.content('.model-facts-long'); - const aliases = qtx('.aka')?.split(/,\s*/); - if (aliases) profile.aliases = aliases; + profile.birthPlace = bio.birth_place; + profile.nationality = bio.nationality; - if (bio.birth_place) profile.birthPlace = bio.birth_place; - if (bio.nationality) profile.nationality = bio.nationality; + profile.measurements = bio.measurements; - if (bio.measurements) { - const [bust, waist, hip] = bio.measurements.split('-'); + profile.height = Number(bio.height?.match(/(\d+)cm/)?.[1]) || null; + profile.weight = Number(bio.weight?.match(/(\d+)kg/i)?.[1]) || null; - if (bust) profile.bust = bust; - if (waist) profile.waist = Number(waist); - if (hip) profile.hip = Number(hip); - } - - if (bio.weight) profile.weight = Number(bio.weight.match(/^\d+/)[0]); - if (bio.height) profile.height = Number(bio.height.match(/^\d+/)[0]); - - if (bio.hair_color) profile.hair = bio.hair_color; - if (bio.eye_color) profile.eye = bio.eye_color; + profile.hairColor = bio.hair_color; + profile.eye = bio.eye_color; if (bio.tattoos) { profile.hasTattoos = true; @@ -149,53 +161,47 @@ function scrapeProfile({ html, q, qa, qtx }) { profile.piercings = bio.piercings; } - profile.avatar = q('.img-pornstar img').dataset.src; - profile.releases = scrapeLatest(html); + profile.avatar = query.img('meta[property="og:image"]', { attribute: 'content' }); // other images are not suitable as avatar + + // profile.releases = scrapeAll(unprint.initAll(query.all('.scenes .card'))); // works for good measure, but no longer in use return profile; } -async function fetchLatest(site, page = 1) { - const { hostname } = new URL(site.url); - - if (hostname.match('private.com')) { - const res = await http.get(`${site.url}/${page}/`); - - return scrapeLatest(res.body.toString(), site); - } - - const res = await http.get(`${site.url}/scenes/${page}/`); - - return scrapeLatest(res.body.toString(), site); -} - -async function fetchScene(url, site) { - const res = await http.get(url); - - return scrapeScene(res.body.toString(), url, site); -} - -async function fetchProfile({ name: actorName }) { +async function searchProfile(actorName) { const actorSearchSlug = slugify(actorName, '+'); const url = `https://www.private.com/search.php?query=${actorSearchSlug}`; - const modelRes = await geta(url, '.model h3 a'); - if (modelRes.ok) { - const actorSlug = slugify(actorName); - const model = modelRes.items.find(({ text }) => slugify(text) === actorSlug); + const res = await unprint.get(url, { headers }); - if (model) { - const res = await get(model.el.href); + return res.context.query.url(`.model a[title="${actorName}"]`); +} - return res.ok ? scrapeProfile(res.item) : res.status; - } +async function fetchProfile({ name: actorName, url }) { + const actorUrl = url || await searchProfile(actorName); + + if (!actorUrl) { + return null; } - return null; + const res = await unprint.get(actorUrl, { headers }); + + if (res.ok) { + return scrapeProfile(res.context); + } + + return res.status; } module.exports = { fetchLatest, - fetchScene, fetchProfile, + scrapeScene: { + scraper: scrapeScene, + headers, + }, + scrapeMovie: { + scraper: scrapeMovie, + headers, + }, };