Refactored Aziani scraper. Improved actor profile update logic.
This commit is contained in:
DebaucheryLibrarian
parent
909dc36569
commit
fbfd52e831
|
|
@ -1325,6 +1325,18 @@ const tags = [
|
|||
name: 'ahegao',
|
||||
slug: 'ahegao',
|
||||
},
|
||||
{
|
||||
name: 'compilation',
|
||||
slug: 'compilation',
|
||||
},
|
||||
{
|
||||
name: 'hotwife',
|
||||
slug: 'hotwife',
|
||||
},
|
||||
{
|
||||
name: 'interview',
|
||||
slug: 'interview',
|
||||
},
|
||||
];
|
||||
|
||||
const aliases = [
|
||||
|
|
@ -1870,6 +1882,10 @@ const aliases = [
|
|||
name: 'double vaginal (dpp)',
|
||||
for: 'dvp',
|
||||
},
|
||||
{
|
||||
name: 'double vagina',
|
||||
for: 'dvp',
|
||||
},
|
||||
{
|
||||
name: 'double pussy penetration',
|
||||
for: 'dvp',
|
||||
|
|
@ -2691,6 +2707,18 @@ const aliases = [
|
|||
name: 'oral cumshot',
|
||||
for: 'cum-in-mouth',
|
||||
},
|
||||
{
|
||||
name: 'compilations',
|
||||
for: 'compilation',
|
||||
},
|
||||
{
|
||||
name: 'hot wife',
|
||||
for: 'hotwife',
|
||||
},
|
||||
{
|
||||
name: 'interviews',
|
||||
for: 'interview',
|
||||
},
|
||||
];
|
||||
|
||||
const priorities = [ // higher index is higher priority
|
||||
|
|
|
|||
|
|
@ -155,10 +155,12 @@ const networks = [
|
|||
{
|
||||
slug: 'aziani',
|
||||
name: 'Aziani',
|
||||
parent: 'gamma',
|
||||
url: 'https://www.aziani.com',
|
||||
parameters: {
|
||||
layout: 'api',
|
||||
areaId: 3,
|
||||
blockId: 114458,
|
||||
scene: 'https://aziani.com',
|
||||
cdn: 'https://c75c0c3063.mjedge.net',
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -849,46 +849,82 @@ const sites = [
|
|||
},
|
||||
},
|
||||
// AZIANI
|
||||
{
|
||||
slug: 'gangbangcreampie',
|
||||
name: 'Gangbang Creampie',
|
||||
url: 'https://www.gangbangcreampie.com',
|
||||
parent: 'aziani',
|
||||
tags: ['gangbang', 'creampie'],
|
||||
parameters: {
|
||||
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'gloryholesecrets',
|
||||
name: 'Glory Hole Secrets',
|
||||
url: 'https://www.gloryholesecrets.com',
|
||||
parent: 'aziani',
|
||||
tags: ['gloryhole'],
|
||||
parameters: {
|
||||
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'aziani',
|
||||
name: 'Aziani',
|
||||
url: 'https://www.aziani.com',
|
||||
url: 'https://www.aziani.com/series/aziani',
|
||||
parent: 'aziani',
|
||||
parameters: {
|
||||
seriesId: 268,
|
||||
areaId: 3,
|
||||
blockId: 114458,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
},
|
||||
/* offline
|
||||
{
|
||||
slug: 'portagloryhole',
|
||||
name: 'Porta Gloryhole',
|
||||
url: 'https://www.portagloryhole.com',
|
||||
slug: '2poles1hole',
|
||||
name: '2 Poles 1 Hole',
|
||||
url: 'https://2poles1hole.com',
|
||||
parent: 'aziani',
|
||||
tags: ['gloryhole'],
|
||||
independent: true,
|
||||
parameters: {
|
||||
areaId: 2,
|
||||
blockId: 114064,
|
||||
seriesId: 107,
|
||||
modelBlockId: 114129,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'creampiled',
|
||||
name: 'CreamPiled',
|
||||
url: 'https://creampiled.com',
|
||||
parent: 'aziani',
|
||||
independent: true,
|
||||
parameters: {
|
||||
areaId: 11,
|
||||
blockId: 115990,
|
||||
seriesId: 436,
|
||||
modelBlockId: 115345,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'popuporgies',
|
||||
name: 'PopUpOrgies',
|
||||
url: 'https://popuporgies.com',
|
||||
parent: 'aziani',
|
||||
independent: true,
|
||||
parameters: {
|
||||
areaId: 8,
|
||||
blockId: 116531,
|
||||
seriesId: 395,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'azianiiron',
|
||||
name: 'Aziani Iron',
|
||||
url: 'https://www.azianiiron.com',
|
||||
url: 'https://aziani.com/azianiiron',
|
||||
parent: 'aziani',
|
||||
parameters: {
|
||||
areaId: 3,
|
||||
blockId: 114458,
|
||||
seriesId: 105,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'mrsaltys',
|
||||
name: 'Mr. Saltys',
|
||||
url: 'https://aziani.com/mrsaltys',
|
||||
parent: 'aziani',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
areaId: 3,
|
||||
blockId: 114458,
|
||||
seriesId: 106,
|
||||
scene: 'https://aziani.com',
|
||||
},
|
||||
},
|
||||
/* offline
|
||||
{
|
||||
slug: 'azianixposed',
|
||||
name: 'Aziani Xposed',
|
||||
|
|
@ -913,6 +949,42 @@ const sites = [
|
|||
url: 'https://www.racqueldevonshire.com',
|
||||
parent: 'aziani',
|
||||
},
|
||||
*/
|
||||
// AZIANI GAMMA
|
||||
{
|
||||
slug: 'gangbangcreampie',
|
||||
name: 'Gangbang Creampie',
|
||||
url: 'https://www.gangbangcreampie.com',
|
||||
parent: 'gamma',
|
||||
tags: ['gangbang', 'creampie'],
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'gloryholesecrets',
|
||||
name: 'Glory Hole Secrets',
|
||||
url: 'https://www.gloryholesecrets.com',
|
||||
parent: 'gamma',
|
||||
tags: ['gloryhole'],
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
|
||||
},
|
||||
},
|
||||
/* different layout
|
||||
{
|
||||
slug: 'portagloryhole',
|
||||
name: 'Porta Gloryhole',
|
||||
url: 'https://www.portagloryhole.com',
|
||||
parent: 'gamma',
|
||||
tags: ['gloryhole'],
|
||||
independent: true,
|
||||
parameters: {
|
||||
scene: 'https://www.portagloryhole.com/scenes',
|
||||
},
|
||||
},
|
||||
*/
|
||||
// BABES
|
||||
{
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ function toBaseActors(actorsOrNames, release) {
|
|||
|
||||
// using top level parent widens the scope too much, e.g. different Gamma sites may not use the same actor database
|
||||
// const entity = getRecursiveParent(release?.entity);
|
||||
const entity = (release?.entity?.indepdendent && release?.entity)
|
||||
const entity = (release?.entity?.independent && release?.entity)
|
||||
|| release?.entity?.parent
|
||||
|| release?.entity
|
||||
|| null;
|
||||
|
|
@ -308,7 +308,7 @@ function curateProfileEntry(profile) {
|
|||
}
|
||||
|
||||
const curatedProfileEntry = {
|
||||
...(profile.update !== false && { id: profile.update }),
|
||||
...(typeof profile.update === 'number' && { id: profile.update }),
|
||||
actor_id: profile.actorId,
|
||||
entity_id: profile.entity?.id || null,
|
||||
date_of_birth: profile.dateOfBirth,
|
||||
|
|
@ -552,21 +552,34 @@ async function upsertProfiles(profiles) {
|
|||
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||
|
||||
const newProfiles = await insertProfiles(newProfileEntries);
|
||||
const newProfiles = newProfileEntries.length > 0
|
||||
? await insertProfiles(newProfileEntries)
|
||||
: [];
|
||||
|
||||
if (argv.force && updatingProfileEntries.length > 0) {
|
||||
const transaction = await knex.transaction();
|
||||
const queries = updatingProfileEntries.map((profileEntry) => knex('actors_profiles')
|
||||
.where('id', profileEntry.id)
|
||||
const queries = updatingProfileEntries.map(async (profileEntry) => knex('actors_profiles')
|
||||
.modify((builder) => {
|
||||
if (profileEntry.id) {
|
||||
builder.where('id', profileEntry.id);
|
||||
} else {
|
||||
builder
|
||||
.where('actor_id', profileEntry.actor_id)
|
||||
.where('entity_id', profileEntry.entity_id);
|
||||
}
|
||||
})
|
||||
.update(profileEntry)
|
||||
.returning(['id', 'actor_id'])
|
||||
.transacting(transaction));
|
||||
|
||||
await Promise.all(queries)
|
||||
.then(transaction.commit)
|
||||
.catch(transaction.rollback);
|
||||
.catch((error) => {
|
||||
logger.error(error.message);
|
||||
return transaction.rollback();
|
||||
});
|
||||
|
||||
logger.info(`Updated ${updatingProfileEntries.length} new actor profiles`);
|
||||
logger.info(`Updated ${updatingProfileEntries.length} actor profiles`);
|
||||
}
|
||||
|
||||
if (profiles.length > 0) {
|
||||
|
|
@ -586,11 +599,13 @@ async function upsertProfiles(profiles) {
|
|||
media_id: profile.avatarMediaId,
|
||||
}));
|
||||
|
||||
if (avatars.length > 0) {
|
||||
await knex('actors_avatars')
|
||||
.insert(avatars)
|
||||
.onConflict()
|
||||
.ignore();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
|
||||
|
|
@ -599,7 +614,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
|||
const profiles = Promise.map(validSources, async (source) => {
|
||||
try {
|
||||
// config may group sources to try until success
|
||||
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
||||
return [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
||||
try {
|
||||
const entity = entitiesBySlug[scraperSlug] || null;
|
||||
|
||||
|
|
@ -846,6 +861,13 @@ async function getOrCreateActors(baseActors, batchId) {
|
|||
OR actors.entry_id = base_actors.entry_id)
|
||||
`);
|
||||
|
||||
const actorIds = existingActors.map((actor) => actor.id);
|
||||
const entityIds = Array.from(new Set(baseActors.map((actor) => actor.entity?.id).filter(Boolean)));
|
||||
|
||||
const existingProfiles = await knex('actors_profiles')
|
||||
.whereIn('actor_id', actorIds)
|
||||
.whereIn('entity_id', entityIds);
|
||||
|
||||
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
|
||||
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
|
||||
...acc,
|
||||
|
|
@ -863,7 +885,7 @@ async function getOrCreateActors(baseActors, batchId) {
|
|||
|
||||
const newActors = await bulkInsert('actors', curatedActorEntries);
|
||||
|
||||
const newActorIdsByEntityIdEntryIdAndSlug = newActors.reduce((acc, actor) => ({
|
||||
const actorIdsByEntityIdEntryIdAndSlug = [...existingActors, ...newActors].reduce((acc, actor) => ({
|
||||
...acc,
|
||||
[actor.entity_id]: {
|
||||
...acc[actor.entity_id],
|
||||
|
|
@ -874,13 +896,26 @@ async function getOrCreateActors(baseActors, batchId) {
|
|||
},
|
||||
}), {});
|
||||
|
||||
const profileIdsByActorIdAndEntityId = existingProfiles.reduce((acc, profile) => ({
|
||||
...acc,
|
||||
[profile.actor_id]: {
|
||||
...acc[profile.actor_id],
|
||||
[profile.entity_id]: profile.id,
|
||||
},
|
||||
}), {});
|
||||
|
||||
const newActorProfiles = await Promise.all(baseActors
|
||||
.filter((actor) => actor.hasProfile)
|
||||
.map((actor) => ({
|
||||
.map((actor) => {
|
||||
const actorId = actorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || actorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug];
|
||||
|
||||
return {
|
||||
...actor,
|
||||
actorId: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
|
||||
}))
|
||||
.filter((actor) => !!actor.id)
|
||||
actorId,
|
||||
update: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
|
||||
};
|
||||
})
|
||||
.filter((actor) => !!actor.actorId)
|
||||
.map((actor) => curateProfile(actor)));
|
||||
|
||||
await storeProfiles(newActorProfiles);
|
||||
|
|
|
|||
|
|
@ -1,147 +1,187 @@
|
|||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const { decode } = require('html-entities');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { get, getAll, initAll, extractDate } = require('../utils/qu');
|
||||
const { feetInchesToCm } = require('../utils/convert');
|
||||
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
|
||||
|
||||
const imageRegex = /-\dx.jpg/;
|
||||
|
||||
function getFallbacks(source) {
|
||||
if (!source || source.includes('join.jpg')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Array.from(new Set([
|
||||
source.replace(imageRegex, '-4x.jpg'),
|
||||
source.replace(imageRegex, '-3x.jpg'),
|
||||
source.replace(imageRegex, '-2x.jpg'),
|
||||
source.replace(imageRegex, '-1x.jpg'),
|
||||
source,
|
||||
]));
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, site) {
|
||||
return scenes.map(({ qu }) => {
|
||||
function scrapeScene(data, channel, parameters) {
|
||||
const release = {};
|
||||
|
||||
release.url = qu.url('a');
|
||||
release.entryId = data.cms_set_id;
|
||||
release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
|
||||
|
||||
release.title = qu.q('h5 a', true);
|
||||
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
|
||||
release.title = data.name;
|
||||
release.description = data.description;
|
||||
|
||||
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(release.url).pathname.match(/trailers\/(.*).html/)?.[1];
|
||||
release.date = unprint.extractDate(data.added_nice, 'YYYY-MM-DD');
|
||||
release.duration = Number(data.lengths.total);
|
||||
|
||||
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ').filter(Boolean);
|
||||
|
||||
const photoCount = qu.q('.stdimage', 'cnt');
|
||||
|
||||
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
|
||||
const source = qu.img('.stdimage', `src${index}_1x`, { origin: site.url });
|
||||
|
||||
return getFallbacks(source);
|
||||
});
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ html, qu }, url, channel) {
|
||||
const release = { url };
|
||||
|
||||
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(url).pathname.match(/trailers\/(.*).html/)?.[1];
|
||||
|
||||
release.title = qu.q('h2', true);
|
||||
release.description = qu.q('p', true);
|
||||
|
||||
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
|
||||
|
||||
release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
|
||||
name: qu.q(actor, null, true),
|
||||
url: qu.url(actor, null),
|
||||
release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models')?.data_values.map((actor) => ({
|
||||
name: actor.name,
|
||||
url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
|
||||
}));
|
||||
|
||||
release.tags = qu.all('.video_categories a', true);
|
||||
release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
|
||||
name: director.name,
|
||||
// url: `${channel.url}/model/${director.slug}?models=${director.name}`,
|
||||
}));
|
||||
|
||||
release.duration = qu.dur('.video_categories + p');
|
||||
release.tags = data.data_types
|
||||
.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
|
||||
.flatMap((tags) => tags.data_values.map((tag) => tag.name));
|
||||
|
||||
release.poster = getFallbacks(qu.img('a img')) || getFallbacks(qu.img('#preview video', 'poster', { origin: channel.url }));
|
||||
release.caps = qu.imgs('.featured-video img', 'src0_1x', { origin: channel.url }).map((source) => getFallbacks(source)).filter(Boolean);
|
||||
const poster = data.preview_formatted.thumb;
|
||||
const teaser = data.preview_formatted.clip?.[0];
|
||||
const trailer = data.preview_formatted.trailer?.formats?.[0]?.content?.[0];
|
||||
|
||||
release.trailer = qu.video('#preview source');
|
||||
if (poster) {
|
||||
release.poster = Object.keys(poster)
|
||||
.filter((key) => poster[key].length > 0)
|
||||
.toSorted((keyA, keyB) => keyB.split('-')[0] - keyA.split('-')[0])
|
||||
.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
|
||||
}
|
||||
|
||||
if (teaser && teaser.fileuri !== trailer?.fileuri) {
|
||||
release.teaser = unprint.prefixUrl(`${teaser.fileuri}?${teaser.signature}`, parameters.cdn);
|
||||
}
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
|
||||
}
|
||||
|
||||
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ el, qu }) {
|
||||
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
const query = new URLSearchParams({
|
||||
cms_area_id: parameters.areaId,
|
||||
cms_block_id: parameters.blockId,
|
||||
count: 100,
|
||||
start: (page - 1) * 100,
|
||||
orderby: 'published_desc',
|
||||
content_type: 'video',
|
||||
// unsure what this does
|
||||
status: 'enabled',
|
||||
cms_set_ids: undefined,
|
||||
data_types: 1,
|
||||
content_count: 1,
|
||||
data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
|
||||
}).toString();
|
||||
|
||||
const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
headers: {
|
||||
Referer: channel.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.data.success) {
|
||||
return res.data.sets.map((data) => scrapeScene(data, channel, parameters));
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity, _baseRelease, { parameters }) {
|
||||
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_set_ids: entryId,
|
||||
cms_area_id: parameters.areaId,
|
||||
cms_block_id: parameters.blockId,
|
||||
content: 1,
|
||||
orderby: 'published_desc',
|
||||
content_type: 'video',
|
||||
// unsure what this does
|
||||
data_types: 1,
|
||||
content_count: 1,
|
||||
}).toString();
|
||||
|
||||
const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||
|
||||
const res = await unprint.get(apiUrl, {
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok && res.data.success) {
|
||||
return scrapeScene(res.data.sets[0], entity, parameters);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile(data, entity, parameters) {
|
||||
const profile = {};
|
||||
|
||||
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
|
||||
const nextNode = nodes[index + 1];
|
||||
const bio = Object.fromEntries(Object.values(data.data_detail_values).map((detail) => [
|
||||
slugify(detail.name, '_'),
|
||||
decode(detail.value || detail.content_formatted || detail.content),
|
||||
]));
|
||||
|
||||
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
|
||||
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
|
||||
profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
|
||||
profile.entryId = data.cms_data_value_id;
|
||||
|
||||
profile.description = data.description;
|
||||
|
||||
profile.gender = bio.gender?.toLowerCase();
|
||||
profile.age = bio.age;
|
||||
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
|
||||
|
||||
profile.measurements = bio.measurements;
|
||||
profile.height = feetInchesToCm(bio.height);
|
||||
profile.foot = femaleFeetUsToEu(bio.foot_size);
|
||||
|
||||
profile.hairColor = bio.hair_color;
|
||||
profile.eyeColor = bio.eye_color;
|
||||
|
||||
const avatar = bio.thumbnail?.image[0];
|
||||
|
||||
if (avatar) {
|
||||
profile.avatar = {
|
||||
src: `${parameters.cdn}${avatar.fileuri}?${avatar.signature}`,
|
||||
expectType: {
|
||||
'application/octet-stream': 'image/jpeg',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
|
||||
if (bio.age) profile.age = Number(bio.age);
|
||||
|
||||
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
|
||||
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
|
||||
|
||||
if (bio.measurements) {
|
||||
const [bust, waist, hip] = bio.measurements.split('-');
|
||||
|
||||
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
|
||||
if (waist) profile.waist = Number(waist);
|
||||
if (hip) profile.hip = Number(hip);
|
||||
}
|
||||
|
||||
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
|
||||
|
||||
if (bio.birth_location) profile.birthPlace = bio.birth_location;
|
||||
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
|
||||
|
||||
if (bio.eye_color) profile.eyes = bio.eye_color;
|
||||
|
||||
const avatar = qu.img('.tac img');
|
||||
profile.avatar = getFallbacks(avatar);
|
||||
|
||||
profile.releases = scrapeAll(initAll(el, '.featured-video'));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page) {
|
||||
const url = `${site.url}/tour/categories/movies_${page}_d.html`;
|
||||
const res = await getAll(url, '.featured-video');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, site);
|
||||
async function fetchProfile({ url }, { entity, parameters }) {
|
||||
if (!url) {
|
||||
// no easy search option
|
||||
return null;
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await get(url, '.trailer');
|
||||
const query = new URLSearchParams({
|
||||
cms_data_value_ids: actorId,
|
||||
cms_block_id: entity.parameters.modelBlockId,
|
||||
cms_data_type_id: 4,
|
||||
}).toString();
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, site);
|
||||
}
|
||||
const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
|
||||
|
||||
return res.status;
|
||||
}
|
||||
const res = await unprint.get(apiUrl, {
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': entity.parameters.areaId,
|
||||
},
|
||||
});
|
||||
|
||||
async function fetchProfile({ name: actorName }, { site }) {
|
||||
const actorSlug = slugify(actorName, '');
|
||||
const url = `${site.url}/tour/models/${actorSlug}.html`;
|
||||
const res = await get(url, '.page-content .row');
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.item);
|
||||
if (res.ok && res.data.success) {
|
||||
return scrapeProfile(res.data.data_values[0], entity, parameters);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ const assylum = require('./assylum');
|
|||
const amateurallure = require('./amateurallure');
|
||||
const americanpornstar = require('./americanpornstar');
|
||||
const amnesiac = require('./amnesiac');
|
||||
const aziani = require('./aziani');
|
||||
const badoink = require('./badoink');
|
||||
const bamvisions = require('./bamvisions');
|
||||
const bang = require('./bang');
|
||||
|
|
@ -94,6 +95,7 @@ const scrapers = {
|
|||
archangel,
|
||||
asiam: modelmedia,
|
||||
assylum,
|
||||
aziani,
|
||||
badoink,
|
||||
bamvisions,
|
||||
bang,
|
||||
|
|
@ -201,6 +203,9 @@ const scrapers = {
|
|||
anilos: nubiles,
|
||||
archangel,
|
||||
asiam: modelmedia,
|
||||
aziani,
|
||||
'2poles1hole': aziani,
|
||||
creampiled: aziani,
|
||||
babes: aylo,
|
||||
babevr: badoink,
|
||||
baddaddypov: fullpornnetwork,
|
||||
|
|
|
|||
|
|
@ -87,15 +87,24 @@ function convertApi(input, fromOrTo, to) {
|
|||
}
|
||||
|
||||
function maleFeetUsToEu(input) {
|
||||
if (!input) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
||||
|
||||
return Math.round((1.27 * size + 29.94) / 0.5) * 0.5; // round to nearest half
|
||||
}
|
||||
|
||||
function femaleFeetUsToEu(input) {
|
||||
if (!input) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
||||
|
||||
return Math.round((1.27 * size + 28.67) / 0.5) * 0.5; // round to nearest half
|
||||
// return Math.round((1.27 * size + 28.67) / 0.5) * 0.5; // round to nearest half
|
||||
return Math.round(1.27 * size + 28.67);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue