DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Updated Jules Jordan scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2023-07-06 00:14:38 +02:00
parent 66439b3b17
commit 0905847ffa
609 changed files with 391 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/deep.js
View File

@@ -69,7 +69,10 @@ async function fetchUnprintScene(scraper, url, entity, baseRelease, options, typ
entity,
baseRelease,
headers: res.headers,
}, options);
include,
beforeFetchScenes: options.beforeFetchScenes,
parameters: options.parameters,
}, options); // options parameter should probably be retired
}
return res.status;

2
src/media.js
View File

@@ -929,7 +929,7 @@ async function associateReleaseMedia(releases, type = 'release') {
logger.error(util.inspect(error.entries, null, null, { color: true }));
}
logger.error(`Failed to store ${type} ${role}: ${error.message}`);
logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'}`);
}
}, Promise.resolve());
}

251
src/scrapers/julesjordan.js
View File

@@ -5,118 +5,13 @@ const Promise = require('bluebird');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const unprint = require('unprint');
const qu = require('../utils/qu');
const http = require('../utils/http');
const { heightToCm } = require('../utils/convert');
const slugify = require('../utils/slugify');
async function fetchPhotos(url) {
const res = await http.get(url);
return res.body.toString();
}
function scrapePhotos(html, type) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const photos = $('.photo_gallery_thumbnail_wrapper .thumbs')
.toArray()
.map((photoElement) => {
const src = $(photoElement).attr('src');
// high res often available in alternative directories, but not always, provide original as fallback
if (type === 'caps') {
return [
src.replace('capthumbs/', 'caps/'),
src,
];
}
return [
src.replace('thumbs/', 'photos/'),
src.replace('thumbs/', '1600watermarked/'),
src.replace('thumbs/', '1280watermarked/'),
src.replace('thumbs/', '1024watermarked/'),
src,
];
});
return photos;
}
async function getPhotosLegacy(entryId, site, type = 'highres', page = 1) {
const albumUrl = `${site.url}/trial/gallery.php?id=${entryId}&type=${type}&page=${page}`;
// logger.warn(`Jules Jordan is using legacy photo scraper for ${albumUrl} (page ${page})`);
const html = await fetchPhotos(albumUrl);
const $ = cheerio.load(html, { normalizeWhitespace: true });
// don't add first URL to pages to prevent unnecessary duplicate request
const photos = scrapePhotos(html, type);
const pages = Array.from(new Set($('.page_numbers a').toArray().map((el) => $(el).attr('href'))));
const otherPhotos = pages
? await Promise.map(pages, async (pageX) => {
const pageUrl = `https://www.julesjordan.com/trial/${pageX}`;
const pageHtml = await fetchPhotos(pageUrl);
return scrapePhotos(pageHtml, type);
}, {
concurrency: 2,
})
: [];
const allPhotos = photos.concat(otherPhotos.flat());
if (allPhotos.length === 0 && type === 'highres') {
// photos not available, try for screencaps instead
return getPhotosLegacy(entryId, site, 'caps', 1);
}
return allPhotos;
}
async function getPhotos(entryId, site, type = 'highres', page = 1) {
const albumUrl = `${site.parameters?.photos || `${site.url}/gallery.php`}?id=${entryId}&type=${type}&page=${page}`;
const res = await http.get(albumUrl);
const html = res.body.toString();
const sourceLines = html.split(/\n/).filter((line) => line.match(/ptx\["\w+"\]/));
const sources = sourceLines.reduce((acc, sourceLine) => {
const quality = sourceLine.match(/\["\w+"\]/)[0].slice(2, -2);
const sourceStart = sourceLine.match(/\/trial|\/tour|\/content/);
if (!sourceStart) return acc;
const source = sourceLine.slice(sourceStart.index, sourceLine.indexOf('.jpg') + 4);
if (!source) return acc;
if (!acc[quality]) acc[quality] = [];
acc[quality].push(`${site.url}${source}`);
return acc;
}, {});
if (type === 'highres') {
if (sources['1600'] && sources['1600'].length > 0) return sources['1600'];
if (sources['1280'] && sources['1280'].length > 0) return sources['1280'];
if (sources['1024'] && sources['1024'].length > 0) return sources['1024'];
if (sources.Thumbs && sources.Thumbs.length > 0) return sources.Thumbs;
// no photos available, try for screencaps instead
return getPhotos(entryId, site, 'caps', 1);
}
if (sources.jpg && sources.jpg.length > 0) return sources.jpg;
if (sources['Video Cap Thumbs'] && sources['Video Cap Thumbs'].length > 0) return sources['Video Cap Thumbs'];
// no screencaps available either, try legacy scraper just in case
return getPhotosLegacy(entryId, site, 'highres', 1);
}
function getEntryId(html) {
const entryId = html.match(/showtagform\((\d+)\)/);
@@ -134,28 +29,28 @@ function getEntryId(html) {
}
function scrapeAll(scenes, site, entryIdFromTitle) {
return scenes.map(({ el, query }) => {
return scenes.map(({ element, query }) => {
const release = {};
const title = query.cnt('.content_img div, .dvd_info > a, a.update_title, a[title] + a[title]') || query.cnt('a[title*=" "]');
const title = query.content('.content_img div, .dvd_info > a, a.update_title, a[title] + a[title], .overlay-text') || query.content('a[title*=" "]');
release.title = title?.slice(0, title.match(/starring:/i)?.index || Infinity).trim();
release.url = query.url('.content_img a, .dvd_info > a, a.update_title, a[title*=" "]');
release.url = query.url('.content_img a, .dvd_info > a, a.update_title, a[title]');
release.date = query.date('.update_date', 'MM/DD/YYYY');
release.entryId = (entryIdFromTitle && slugify(release.title)) || el.dataset.setid || query.q('.rating_box')?.dataset.id;
release.entryId = (entryIdFromTitle && slugify(release.title)) || element.dataset.setid || query.element('.rating_box')?.dataset.id || query.attribute('a img', 'id')?.match(/set-target-(\d+)/)?.[1];
release.actors = query.all('.content_img .update_models a, .update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
const dvdPhotos = query.imgs('.dvd_preview_thumb');
const photoCount = Number(query.q('a img.thumbs', 'cnt')) || 1;
const photoCount = Number(query.attribute('a img.thumbs', 'cnt')) || 1;
[release.poster, ...release.photos] = dvdPhotos.length
? dvdPhotos
: Array.from({ length: photoCount }).map((value, index) => {
const src = query.img('a img.thumbs', `src${index}_1x`) || query.img('a img.thumbs', `src${index}`) || query.img('a img.thumbs');
const src = query.img('a img.thumbs', { attribute: `src${index}_1x` }) || query.img('a img.thumbs', { attribute: `src${index}` }) || query.img('a img.thumbs');
const prefixedSrc = qu.prefixUrl(src, site.url);
if (src) {
@@ -239,56 +134,105 @@ function scrapeUpcoming(html, site) {
});
}
async function scrapeScene({ html, query }, url, site, options) {
function extractLegacyTrailer(html, context) {
const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) {
return trailerLines.map((trailerLine) => {
// const src = trailerLine.match(/path:"([\w-:/.&=?%]+)"/)?.[1];
const src = trailerLine.match(/path:"(.+)"/)?.[1];
const quality = trailerLine.match(/movie_height:'(\d+)/)?.[1];
return src && {
src: /^http/.test(src) ? src : `${context.entity.url}${src}`,
quality: quality && Number(quality.replace('558', '540')),
};
}).filter(Boolean);
}
return null;
}
const qualities = [
'photos',
'1600watermarked',
'1280watermarked',
'1024watermarked',
'thumbs',
];
function getPhotos(query, release, context) {
// https://thumbs.julesjordan.com/members/content//upload/dl03/julesjordan/whitney_wright_dredd/1024watermarked/whitney_wright_julesjordan.com-20.jpg
// https://thumbs.julesjordan.com/members/content//upload/dl03/julesjordan/bambi_barton_manuel_ferrara/1024watermarked/bambi_barton_julesjordan_com-13.jpg
if (!release.actors?.length > 0) {
return null;
}
const photoCount = query.number('//div[contains(@class, "title-heading-content")][contains(text(), "Photos")]');
if (photoCount) {
// slug actor order is not always the same as actor list order, prefer trailer slug if available
const path = query.dataset('.movieformat_button', 'src')?.match(/:(.*)_trailer/)?.[1] || release.actors.map((actor) => slugify(actor.name || actor, '_')).join('_');
const derivedActorSlug = path.replace(`_${release.actors.slice(1).map(({ name }) => slugify(name, '_'))}`, '');
const actorSlug = derivedActorSlug === path // no replacement took place, so the slug is likely invalid
? slugify(release.actors[0].name || release.actors[0], '_')
: derivedActorSlug;
return Array.from({ length: photoCount }, (value, index) => qualities
.flatMap((quality) => [
`https://thumbs.${context.entity.slug}.com/trial/content//upload/dl03/${context.entity.slug}/${path}/${quality}/${actorSlug}_${context.entity.slug}_com-${index + 1}.jpg`,
`https://thumbs.${context.entity.slug}.com/trial/content//upload/dl03/${context.entity.slug}/${path}/${quality}/${actorSlug}_${context.entity.slug}.com-${index + 1}.jpg`, // .com instead of _com
]));
}
return null;
}
async function scrapeScene({ html, query }, context) {
const release = {};
release.entryId = getEntryId(html);
release.title = query.cnt('.title_bar_hilite');
release.description = query.cnt('.update_description');
release.title = query.content('.title_bar_hilite, .movie_title');
release.description = query.content('.update_description') || query.text('//div[./span[contains(text(), "Description")]]');
release.entryId = context.entity.parameters?.entryIdFromTitle ? slugify(release.title) : getEntryId(html);
release.date = query.date('.update_date', 'MM/DD/YYYY', null, 'innerHTML');
release.date = query.date(['.update_date', '//div[./span[contains(text(), "Date")]]'], 'MM/DD/YYYY');
release.actors = query.all('.backgroundcolor_info > .update_models a, .item .update_models a').map((actorEl) => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null),
release.actors = query.all('.backgroundcolor_info > .update_models a, .item .update_models a, .player-scene-description .update_models a').map((actorEl) => ({
name: unprint.query.content(actorEl),
url: unprint.query.url(actorEl, null),
}));
release.tags = query.cnts('.update_tags a');
release.tags = query.contents('.update_tags a, .player-scene-description a[href*="/categories"]');
release.director = release.tags?.find((tag) => ['mike john', 'van styles'].includes(tag?.trim().toLowerCase()));
const posterPath = html.match(/useimage = "(.*)"/)?.[1];
const posterPath = query.poster('#video-player') || html.match(/useimage = "(.*)"/)?.[1];
if (posterPath) {
const poster = /^http/.test(posterPath) ? posterPath : `${site.url}${posterPath}`;
const poster = /^http/.test(posterPath) ? posterPath : `${context.entity.url}${posterPath}`;
if (poster) {
release.poster = {
src: poster,
referer: site.url,
referer: context.entity.url,
};
}
}
if (options.includeTrailers && site.slug !== 'manuelferrara') {
const trailerLines = html.split('\n').filter((line) => /movie\["trailer\w*"\]\[/i.test(line));
if (trailerLines.length) {
release.trailer = trailerLines.map((trailerLine) => {
// const src = trailerLine.match(/path:"([\w-:/.&=?%]+)"/)?.[1];
const src = trailerLine.match(/path:"(.+)"/)?.[1];
const quality = trailerLine.match(/movie_height:'(\d+)/)?.[1];
return src && {
src: /^http/.test(src) ? src : `${site.url}${src}`,
quality: quality && Number(quality.replace('558', '540')),
};
}).filter(Boolean);
}
if (query.exists('source[data-bitrate="trailer"]')) {
release.trailer = [
query.video('source[data-bitrate="trailer_1080" i]'),
query.video('source[data-bitrate="trailer_720" i]'),
query.video('source[data-bitrate="trailer" i]'), // also seems to be 720p
query.video('source[data-bitrate="trailer_mobile" i]'), // also seems to be 720p
];
} else if (context.include.trailers && context.entity.slug !== 'manuelferrara') {
release.trailer = extractLegacyTrailer(html, context);
}
if (options.includePhotos) {
release.photos = await getPhotos(release.entryId, site);
}
// release.photos = async () => await getPhotos(release.entryId, context.entity); // probably no longer works on any site
// release.photos = query.imgs('#images img');
release.photos = getPhotos(query, release, context);
if (query.exists('.update_dvds a')) {
release.movie = {
@@ -381,9 +325,13 @@ async function fetchLatest(site, page = 1, include, preData, entryIdFromTitle =
: `${site.url}/trial/categories/movies_${page}_d.html`;
// const res = await http.get(url);
const res = await qu.getAll(url, '.update_details');
const res = await unprint.get(url, { selectAll: '.update_details, .grid-item' });
return res.ok ? scrapeAll(res.items, site, entryIdFromTitle) : res.status;
if (res.ok) {
return scrapeAll(res.context, site, typeof site.parameters?.entryIdFromTitle === 'boolean' ? site.parameters.entryIdFromTitle : entryIdFromTitle);
}
return res.status;
}
async function fetchUpcoming(site) {
@@ -399,12 +347,6 @@ async function fetchUpcoming(site) {
return res.statusCode;
}
async function fetchScene(url, site, baseRelease, include) {
const res = await qu.get(url);
return res.ok ? scrapeScene(res.item, url, site, include) : res.status;
}
async function fetchMovie(url, site) {
const res = await qu.get(url);
@@ -447,5 +389,8 @@ module.exports = {
fetchMovie,
fetchProfile,
fetchUpcoming,
fetchScene,
scrapeScene: {
scraper: scrapeScene,
unprint: true,
},
};

11
src/tools/knex-error.js Normal file
View File

@@ -0,0 +1,11 @@
const knex = require('../knex');
async function init() {
try {
await knex.raw('INSERT INTO actors_profiles (actor_id) VALUES (1000000000) ON CONFLICT DO NOTHING RETURNING *;');
} catch (error) {
console.log(error);
}
}
init();

10
src/utils/bulk-insert.js
View File

@@ -2,6 +2,7 @@
const knex = require('../knex');
const chunk = require('./chunk');
const logger = require('../logger')(__filename);
async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
if (items.length === 0) {
@@ -26,9 +27,14 @@ async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
query: knex(table).insert(chunkItems),
}).transacting(transaction));
const responses = await Promise.all(queries);
try {
const responses = await Promise.all(queries);
return responses.flat().map((response) => response.rows).flat();
return responses.flat().map((response) => response.rows).flat();
} catch (error) {
logger.error(`Failed bulk insert: ${error.message} (${error.detail})`);
throw error;
}
});
}