Refactored Private scraper.
This commit is contained in:
DebaucheryLibrarian
parent
8167809bd5
commit
01706d2648
|
|
@ -83,39 +83,41 @@ const actorsFields = `
|
|||
scenes int
|
||||
`;
|
||||
|
||||
exports.up = async (knex) => {
|
||||
exports.up = async () => {
|
||||
try {
|
||||
await utilsApi.sql(`create table scenes (${scenesFields})`);
|
||||
await utilsApi.sql(`create table if not exists scenes (${scenesFields})`);
|
||||
|
||||
await utilsApi.sql(`create table scenes_stashed (
|
||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
||||
scene_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql(`create table movies (${moviesFields})`);
|
||||
await utilsApi.sql(`create table if not exists movies (${moviesFields})`);
|
||||
|
||||
await utilsApi.sql(`create table movies_stashed (
|
||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
||||
movie_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql(`create table actors (${actorsFields}) min_prefix_len='3'`);
|
||||
await utilsApi.sql(`create table if not exists actors (${actorsFields}) min_prefix_len='3'`);
|
||||
|
||||
await utilsApi.sql(`create table actors_stashed (
|
||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
||||
actor_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
/*
|
||||
await knex.schema.alterTable('stashes_scenes', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_movies', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_actors', (table) => table.increments('id'));
|
||||
await knex.schema.alterTable('stashes_series', (table) => table.increments('id'));
|
||||
*/
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,138 +1,150 @@
|
|||
'use strict';
|
||||
|
||||
/* eslint-disable newline-per-chained-call */
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const { get, geta } = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
const http = require('../utils/http');
|
||||
|
||||
const headers = {
|
||||
'Accept-Language': 'en-US,en', // will translate to server language if not specified
|
||||
};
|
||||
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('h3 a');
|
||||
release.entryId = new URL(release.url).pathname.split('/').at(-1);
|
||||
|
||||
release.title = query.content('h3 a')
|
||||
|| query.attribute('a img', 'alt')?.split(/:(.*)/)[1]?.trim()
|
||||
|| query.attribute('a img', 'alt');
|
||||
|
||||
release.date = query.date('.scene-date', ['MM/DD/YYYY', 'YYYY-MM-DD']);
|
||||
|
||||
release.actors = query.all('.scene-models a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.poster = query.img('a img');
|
||||
release.teaser = query.video('.mini_video_player source');
|
||||
|
||||
console.log(release);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const { hostname } = new URL(site.url);
|
||||
|
||||
if (hostname.match('private.com')) {
|
||||
const res = await unprint.get(`${site.url}/${page}/`, { selectAll: '.content-wrapper .scene', headers });
|
||||
|
||||
return scrapeAll(res.context, site);
|
||||
}
|
||||
|
||||
const res = await unprint.get(`${site.url}/scenes/${page}/`, { selectAll: '.content-wrapper .scene', headers });
|
||||
|
||||
return scrapeAll(res.context, site);
|
||||
}
|
||||
|
||||
async function getPhotos(entryId, site) {
|
||||
const { hostname } = new URL(site.url);
|
||||
|
||||
const res = await http.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`);
|
||||
const html = res.body.toString();
|
||||
const res = await unprint.get(`https://${hostname}/gallery.php?type=highres&id=${entryId}`);
|
||||
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const photos = $('a.fakethumb').map((photoIndex, photoElement) => $(photoElement).attr('data-src') || $(photoElement).attr('href')).toArray();
|
||||
|
||||
return photos;
|
||||
}
|
||||
|
||||
function scrapeLatest(html, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const sceneElements = $('.content-wrapper .scene').toArray();
|
||||
|
||||
return sceneElements.map((element) => {
|
||||
const sceneLinkElement = $(element).find('h3 a');
|
||||
const thumbnailElement = $(element).find('a img');
|
||||
|
||||
const url = sceneLinkElement.attr('href');
|
||||
// const title = sceneLinkElement.text();
|
||||
const entryId = url.split('/').slice(-1)[0];
|
||||
|
||||
const titleText = thumbnailElement.attr('alt');
|
||||
const title = titleText.slice(titleText.indexOf(':') + 1).trim();
|
||||
|
||||
const date = moment.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
|
||||
|
||||
const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
const likes = Number($(element).find('.scene-votes').text());
|
||||
|
||||
const photoCount = Number(thumbnailElement.attr('thumbs_num'));
|
||||
const poster = thumbnailElement.attr('src');
|
||||
const photos = Array.from({ length: photoCount }, (val, index) => thumbnailElement.attr(`src${index + 1}`));
|
||||
|
||||
const scene = {
|
||||
url,
|
||||
entryId,
|
||||
title,
|
||||
actors,
|
||||
date,
|
||||
poster,
|
||||
photos,
|
||||
rating: {
|
||||
likes,
|
||||
},
|
||||
site,
|
||||
};
|
||||
|
||||
return scene;
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const release = { url };
|
||||
|
||||
[release.entryId] = url.split('/').slice(-1);
|
||||
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
|
||||
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
|
||||
|
||||
release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
|
||||
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
|
||||
|
||||
if (timestamp) {
|
||||
const [minutes, seconds] = timestamp.match(/\d+/g);
|
||||
release.duration = Number(minutes) * 60 + Number(seconds);
|
||||
if (res.ok) {
|
||||
return res.context.query.urls('#lightgallery a.item');
|
||||
}
|
||||
|
||||
release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
release.likes = Number($('.content-desc #social-actions #likes').text());
|
||||
return [];
|
||||
}
|
||||
|
||||
const posterScript = $('script:contains(poster)').html();
|
||||
const posterLink = posterScript?.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
|
||||
release.poster = $('meta[property="og:image"]').attr('content') || posterLink || $('#trailer_player_finished img').attr('src');
|
||||
async function scrapeScene({ query }, { url, entity }) {
|
||||
const release = {};
|
||||
|
||||
const trailer = $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src');
|
||||
release.entryId = new URL(url).pathname.split('/').at(-1);
|
||||
release.channel = slugify(query.content('.title-site'), '');
|
||||
|
||||
if (trailer) release.trailer = { src: trailer };
|
||||
release.title = query.content(['.title-zone h1', '.content-desc h1']);
|
||||
release.description = query.content('meta[itemprop="description"]', { attribute: 'content' }) || query.content('#description-section');
|
||||
|
||||
release.photos = await getPhotos(release.entryId, site);
|
||||
release.movie = $('a[data-track="FULL MOVIE"]').attr('href');
|
||||
release.date = query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' });
|
||||
release.duration = query.duration('meta[itemprop="duration"]', { attribute: 'content' });
|
||||
|
||||
const siteElement = $('.content-wrapper .logos-sites a');
|
||||
if (siteElement) release.channel = slugify(siteElement.text(), '');
|
||||
release.actors = query.all('.title-zone .tag-models a, .scene-models-list a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.tags = query.contents('.title-zone .tag-tags, .scene-tags a');
|
||||
|
||||
release.poster = query.img('meta[itemprop="thumbnailUrl"]', { attribute: 'content' }) || query.img('#trailer_player_finished img');
|
||||
release.photos = await getPhotos(release.entryId, entity);
|
||||
release.trailer = query.url('meta[itemprop="contentURL"]', { attribute: 'content' }) || query.video('#trailer_player source');
|
||||
|
||||
const movieUrl = query.url('.content-desc a[href*="movie/"]');
|
||||
|
||||
if (movieUrl) {
|
||||
release.movie = {
|
||||
url: movieUrl,
|
||||
entryId: new URL(movieUrl).pathname.match(/movie\/(\d+)/)?.[1],
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ html, q, qa, qtx }) {
|
||||
async function scrapeMovie({ query }, { url }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/movie\/(\d+)/)[1];
|
||||
|
||||
release.title = query.content('.dvds-wrapper h1');
|
||||
release.description = query.content('meta[itemprop="description"]', { attribute: 'content' }) || query.content('p.sinopsys');
|
||||
|
||||
release.date = query.date('.date-duration [itemprop="datePublished"]', 'YYYY-MM-DD')
|
||||
|| query.date('.date-duration em + span', 'DD/MM/YYYY')
|
||||
|| query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' }); // upload date does not match release date
|
||||
|
||||
release.duration = query.number('.date-duration', { match: /(\d+) min/, matchIndex: 1 }) * 60 || null;
|
||||
|
||||
release.actors = query.all('.dvd-performers a').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.poster = query.img('meta[itemprop="thumbnailUrl"]', { attribute: 'content' }) || query.img('#trailer_player_finished img');
|
||||
release.covers = query.imgs('.dvds-picture img');
|
||||
|
||||
release.trailer = query.url('meta[itemprop="contentURL"]', { attribute: 'content' }) || query.video('#trailer_player source');
|
||||
|
||||
release.scenes = scrapeAll(unprint.initAll(query.all('.scenes .card')));
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
const bio = qa('.model-facts li:not(.model-facts-long)', true).reduce((acc, fact) => {
|
||||
const [key, value] = fact.split(':');
|
||||
const trimmedValue = value.trim();
|
||||
const bioKeys = query.contents('.bio-pornstar .model-facts:last-child em');
|
||||
const bioValues = query.texts('.bio-pornstar .model-facts:last-child li');
|
||||
|
||||
if (trimmedValue.length === 0 || trimmedValue === '-') return acc;
|
||||
return { ...acc, [slugify(key, '_')]: trimmedValue };
|
||||
}, {});
|
||||
const bio = Object.fromEntries(bioKeys.map((key, index) => [slugify(key, '_'), bioValues[index]]).filter(([, value]) => value !== '-'));
|
||||
|
||||
const description = q('.model-facts-long', true);
|
||||
if (description) profile.description = description;
|
||||
profile.description = query.content('.model-facts-long');
|
||||
|
||||
const aliases = qtx('.aka')?.split(/,\s*/);
|
||||
if (aliases) profile.aliases = aliases;
|
||||
profile.birthPlace = bio.birth_place;
|
||||
profile.nationality = bio.nationality;
|
||||
|
||||
if (bio.birth_place) profile.birthPlace = bio.birth_place;
|
||||
if (bio.nationality) profile.nationality = bio.nationality;
|
||||
profile.measurements = bio.measurements;
|
||||
|
||||
if (bio.measurements) {
|
||||
const [bust, waist, hip] = bio.measurements.split('-');
|
||||
profile.height = Number(bio.height?.match(/(\d+)cm/)?.[1]) || null;
|
||||
profile.weight = Number(bio.weight?.match(/(\d+)kg/i)?.[1]) || null;
|
||||
|
||||
if (bust) profile.bust = bust;
|
||||
if (waist) profile.waist = Number(waist);
|
||||
if (hip) profile.hip = Number(hip);
|
||||
}
|
||||
|
||||
if (bio.weight) profile.weight = Number(bio.weight.match(/^\d+/)[0]);
|
||||
if (bio.height) profile.height = Number(bio.height.match(/^\d+/)[0]);
|
||||
|
||||
if (bio.hair_color) profile.hair = bio.hair_color;
|
||||
if (bio.eye_color) profile.eye = bio.eye_color;
|
||||
profile.hairColor = bio.hair_color;
|
||||
profile.eye = bio.eye_color;
|
||||
|
||||
if (bio.tattoos) {
|
||||
profile.hasTattoos = true;
|
||||
|
|
@ -149,53 +161,47 @@ function scrapeProfile({ html, q, qa, qtx }) {
|
|||
profile.piercings = bio.piercings;
|
||||
}
|
||||
|
||||
profile.avatar = q('.img-pornstar img').dataset.src;
|
||||
profile.releases = scrapeLatest(html);
|
||||
profile.avatar = query.img('meta[property="og:image"]', { attribute: 'content' }); // other images are not suitable as avatar
|
||||
|
||||
// profile.releases = scrapeAll(unprint.initAll(query.all('.scenes .card'))); // works for good measure, but no longer in use
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const { hostname } = new URL(site.url);
|
||||
|
||||
if (hostname.match('private.com')) {
|
||||
const res = await http.get(`${site.url}/${page}/`);
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
}
|
||||
|
||||
const res = await http.get(`${site.url}/scenes/${page}/`);
|
||||
|
||||
return scrapeLatest(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await http.get(url);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site);
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }) {
|
||||
async function searchProfile(actorName) {
|
||||
const actorSearchSlug = slugify(actorName, '+');
|
||||
const url = `https://www.private.com/search.php?query=${actorSearchSlug}`;
|
||||
const modelRes = await geta(url, '.model h3 a');
|
||||
|
||||
if (modelRes.ok) {
|
||||
const actorSlug = slugify(actorName);
|
||||
const model = modelRes.items.find(({ text }) => slugify(text) === actorSlug);
|
||||
const res = await unprint.get(url, { headers });
|
||||
|
||||
if (model) {
|
||||
const res = await get(model.el.href);
|
||||
return res.context.query.url(`.model a[title="${actorName}"]`);
|
||||
}
|
||||
|
||||
return res.ok ? scrapeProfile(res.item) : res.status;
|
||||
}
|
||||
async function fetchProfile({ name: actorName, url }) {
|
||||
const actorUrl = url || await searchProfile(actorName);
|
||||
|
||||
if (!actorUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
const res = await unprint.get(actorUrl, { headers });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.context);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
scrapeScene: {
|
||||
scraper: scrapeScene,
|
||||
headers,
|
||||
},
|
||||
scrapeMovie: {
|
||||
scraper: scrapeMovie,
|
||||
headers,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue