DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Refactored Aziani scraper. Improved actor profile update logic.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2024-11-24 06:10:21 +01:00
parent 909dc36569
commit fbfd52e831
7 changed files with 354 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
src/scrapers/aziani.js
View File

@@ -1,147 +1,187 @@
'use strict';
const unprint = require('unprint');
const { decode } = require('html-entities');
const slugify = require('../utils/slugify');
const { get, getAll, initAll, extractDate } = require('../utils/qu');
const { feetInchesToCm } = require('../utils/convert');
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
const imageRegex = /-\dx.jpg/;
function scrapeScene(data, channel, parameters) {
const release = {};
function getFallbacks(source) {
if (!source || source.includes('join.jpg')) {
return null;
}
release.entryId = data.cms_set_id;
release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
return Array.from(new Set([
source.replace(imageRegex, '-4x.jpg'),
source.replace(imageRegex, '-3x.jpg'),
source.replace(imageRegex, '-2x.jpg'),
source.replace(imageRegex, '-1x.jpg'),
source,
]));
}
release.title = data.name;
release.description = data.description;
function scrapeAll(scenes, site) {
return scenes.map(({ qu }) => {
const release = {};
release.date = unprint.extractDate(data.added_nice, 'YYYY-MM-DD');
release.duration = Number(data.lengths.total);
release.url = qu.url('a');
release.title = qu.q('h5 a', true);
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(release.url).pathname.match(/trailers\/(.*).html/)?.[1];
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ').filter(Boolean);
const photoCount = qu.q('.stdimage', 'cnt');
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
const source = qu.img('.stdimage', `src${index}_1x`, { origin: site.url });
return getFallbacks(source);
});
return release;
});
}
function scrapeScene({ html, qu }, url, channel) {
const release = { url };
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(url).pathname.match(/trailers\/(.*).html/)?.[1];
release.title = qu.q('h2', true);
release.description = qu.q('p', true);
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
name: qu.q(actor, null, true),
url: qu.url(actor, null),
release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models')?.data_values.map((actor) => ({
name: actor.name,
url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
}));
release.tags = qu.all('.video_categories a', true);
release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
name: director.name,
// url: `${channel.url}/model/${director.slug}?models=${director.name}`,
}));
release.duration = qu.dur('.video_categories + p');
release.tags = data.data_types
.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
.flatMap((tags) => tags.data_values.map((tag) => tag.name));
release.poster = getFallbacks(qu.img('a img')) || getFallbacks(qu.img('#preview video', 'poster', { origin: channel.url }));
release.caps = qu.imgs('.featured-video img', 'src0_1x', { origin: channel.url }).map((source) => getFallbacks(source)).filter(Boolean);
const poster = data.preview_formatted.thumb;
const teaser = data.preview_formatted.clip?.[0];
const trailer = data.preview_formatted.trailer?.formats?.[0]?.content?.[0];
release.trailer = qu.video('#preview source');
if (poster) {
release.poster = Object.keys(poster)
.filter((key) => poster[key].length > 0)
.toSorted((keyA, keyB) => keyB.split('-')[0] - keyA.split('-')[0])
.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
}
if (teaser && teaser.fileuri !== trailer?.fileuri) {
release.teaser = unprint.prefixUrl(`${teaser.fileuri}?${teaser.signature}`, parameters.cdn);
}
if (trailer) {
release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
}
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
return release;
}
function scrapeProfile({ el, qu }) {
const profile = {};
async function fetchLatest(channel, page = 1, { parameters }) {
const query = new URLSearchParams({
cms_area_id: parameters.areaId,
cms_block_id: parameters.blockId,
count: 100,
start: (page - 1) * 100,
orderby: 'published_desc',
content_type: 'video',
// unsure what this does
status: 'enabled',
cms_set_ids: undefined,
data_types: 1,
content_count: 1,
data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
}).toString();
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
const nextNode = nodes[index + 1];
const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
}
const res = await unprint.get(url, {
headers: {
Referer: channel.url,
'x-nats-cms-area-id': parameters.areaId,
},
});
return acc;
}, {});
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
if (bio.age) profile.age = Number(bio.age);
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
if (bio.measurements) {
const [bust, waist, hip] = bio.measurements.split('-');
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
if (waist) profile.waist = Number(waist);
if (hip) profile.hip = Number(hip);
if (res.ok && res.data.success) {
return res.data.sets.map((data) => scrapeScene(data, channel, parameters));
}
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
return res.status;
}
if (bio.birth_location) profile.birthPlace = bio.birth_location;
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
async function fetchScene(url, entity, _baseRelease, { parameters }) {
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
if (bio.eye_color) profile.eyes = bio.eye_color;
const query = new URLSearchParams({
cms_set_ids: entryId,
cms_area_id: parameters.areaId,
cms_block_id: parameters.blockId,
content: 1,
orderby: 'published_desc',
content_type: 'video',
// unsure what this does
data_types: 1,
content_count: 1,
}).toString();
const avatar = qu.img('.tac img');
profile.avatar = getFallbacks(avatar);
const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
profile.releases = scrapeAll(initAll(el, '.featured-video'));
const res = await unprint.get(apiUrl, {
headers: {
Referer: entity.url,
'x-nats-cms-area-id': parameters.areaId,
},
});
if (res.ok && res.data.success) {
return scrapeScene(res.data.sets[0], entity, parameters);
}
return res.status;
}
function scrapeProfile(data, entity, parameters) {
const profile = {};
const bio = Object.fromEntries(Object.values(data.data_detail_values).map((detail) => [
slugify(detail.name, '_'),
decode(detail.value || detail.content_formatted || detail.content),
]));
profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
profile.entryId = data.cms_data_value_id;
profile.description = data.description;
profile.gender = bio.gender?.toLowerCase();
profile.age = bio.age;
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
profile.measurements = bio.measurements;
profile.height = feetInchesToCm(bio.height);
profile.foot = femaleFeetUsToEu(bio.foot_size);
profile.hairColor = bio.hair_color;
profile.eyeColor = bio.eye_color;
const avatar = bio.thumbnail?.image[0];
if (avatar) {
profile.avatar = {
src: `${parameters.cdn}${avatar.fileuri}?${avatar.signature}`,
expectType: {
'application/octet-stream': 'image/jpeg',
},
};
}
return profile;
}
async function fetchLatest(site, page) {
const url = `${site.url}/tour/categories/movies_${page}_d.html`;
const res = await getAll(url, '.featured-video');
if (res.ok) {
return scrapeAll(res.items, site);
async function fetchProfile({ url }, { entity, parameters }) {
if (!url) {
// no easy search option
return null;
}
return res.status;
}
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
async function fetchScene(url, site) {
const res = await get(url, '.trailer');
const query = new URLSearchParams({
cms_data_value_ids: actorId,
cms_block_id: entity.parameters.modelBlockId,
cms_data_type_id: 4,
}).toString();
if (res.ok) {
return scrapeScene(res.item, url, site);
}
const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
return res.status;
}
const res = await unprint.get(apiUrl, {
headers: {
Referer: entity.url,
'x-nats-cms-area-id': entity.parameters.areaId,
},
});
async function fetchProfile({ name: actorName }, { site }) {
const actorSlug = slugify(actorName, '');
const url = `${site.url}/tour/models/${actorSlug}.html`;
const res = await get(url, '.page-content .row');
if (res.ok) {
return scrapeProfile(res.item);
if (res.ok && res.data.success) {
return scrapeProfile(res.data.data_values[0], entity, parameters);
}
return res.status;

5
src/scrapers/scrapers.js
View File

@@ -7,6 +7,7 @@ const assylum = require('./assylum');
const amateurallure = require('./amateurallure');
const americanpornstar = require('./americanpornstar');
const amnesiac = require('./amnesiac');
const aziani = require('./aziani');
const badoink = require('./badoink');
const bamvisions = require('./bamvisions');
const bang = require('./bang');
@@ -94,6 +95,7 @@ const scrapers = {
archangel,
asiam: modelmedia,
assylum,
aziani,
badoink,
bamvisions,
bang,
@@ -201,6 +203,9 @@ const scrapers = {
anilos: nubiles,
archangel,
asiam: modelmedia,
aziani,
'2poles1hole': aziani,
creampiled: aziani,
babes: aylo,
babevr: badoink,
baddaddypov: fullpornnetwork,