Refactored Aziani scraper. Improved actor profile update logic.
This commit is contained in:
parent
909dc36569
commit
fbfd52e831
|
@ -1325,6 +1325,18 @@ const tags = [
|
||||||
name: 'ahegao',
|
name: 'ahegao',
|
||||||
slug: 'ahegao',
|
slug: 'ahegao',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'compilation',
|
||||||
|
slug: 'compilation',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'hotwife',
|
||||||
|
slug: 'hotwife',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'interview',
|
||||||
|
slug: 'interview',
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const aliases = [
|
const aliases = [
|
||||||
|
@ -1870,6 +1882,10 @@ const aliases = [
|
||||||
name: 'double vaginal (dpp)',
|
name: 'double vaginal (dpp)',
|
||||||
for: 'dvp',
|
for: 'dvp',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'double vagina',
|
||||||
|
for: 'dvp',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'double pussy penetration',
|
name: 'double pussy penetration',
|
||||||
for: 'dvp',
|
for: 'dvp',
|
||||||
|
@ -2691,6 +2707,18 @@ const aliases = [
|
||||||
name: 'oral cumshot',
|
name: 'oral cumshot',
|
||||||
for: 'cum-in-mouth',
|
for: 'cum-in-mouth',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'compilations',
|
||||||
|
for: 'compilation',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'hot wife',
|
||||||
|
for: 'hotwife',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'interviews',
|
||||||
|
for: 'interview',
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const priorities = [ // higher index is higher priority
|
const priorities = [ // higher index is higher priority
|
||||||
|
|
|
@ -155,10 +155,12 @@ const networks = [
|
||||||
{
|
{
|
||||||
slug: 'aziani',
|
slug: 'aziani',
|
||||||
name: 'Aziani',
|
name: 'Aziani',
|
||||||
parent: 'gamma',
|
|
||||||
url: 'https://www.aziani.com',
|
url: 'https://www.aziani.com',
|
||||||
parameters: {
|
parameters: {
|
||||||
layout: 'api',
|
areaId: 3,
|
||||||
|
blockId: 114458,
|
||||||
|
scene: 'https://aziani.com',
|
||||||
|
cdn: 'https://c75c0c3063.mjedge.net',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
@ -849,46 +849,82 @@ const sites = [
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// AZIANI
|
// AZIANI
|
||||||
{
|
|
||||||
slug: 'gangbangcreampie',
|
|
||||||
name: 'Gangbang Creampie',
|
|
||||||
url: 'https://www.gangbangcreampie.com',
|
|
||||||
parent: 'aziani',
|
|
||||||
tags: ['gangbang', 'creampie'],
|
|
||||||
parameters: {
|
|
||||||
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
slug: 'gloryholesecrets',
|
|
||||||
name: 'Glory Hole Secrets',
|
|
||||||
url: 'https://www.gloryholesecrets.com',
|
|
||||||
parent: 'aziani',
|
|
||||||
tags: ['gloryhole'],
|
|
||||||
parameters: {
|
|
||||||
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
slug: 'aziani',
|
slug: 'aziani',
|
||||||
name: 'Aziani',
|
name: 'Aziani',
|
||||||
url: 'https://www.aziani.com',
|
url: 'https://www.aziani.com/series/aziani',
|
||||||
parent: 'aziani',
|
parent: 'aziani',
|
||||||
|
parameters: {
|
||||||
|
seriesId: 268,
|
||||||
|
areaId: 3,
|
||||||
|
blockId: 114458,
|
||||||
|
scene: 'https://aziani.com',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: '2poles1hole',
|
||||||
|
name: '2 Poles 1 Hole',
|
||||||
|
url: 'https://2poles1hole.com',
|
||||||
|
parent: 'aziani',
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
areaId: 2,
|
||||||
|
blockId: 114064,
|
||||||
|
seriesId: 107,
|
||||||
|
modelBlockId: 114129,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'creampiled',
|
||||||
|
name: 'CreamPiled',
|
||||||
|
url: 'https://creampiled.com',
|
||||||
|
parent: 'aziani',
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
areaId: 11,
|
||||||
|
blockId: 115990,
|
||||||
|
seriesId: 436,
|
||||||
|
modelBlockId: 115345,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'popuporgies',
|
||||||
|
name: 'PopUpOrgies',
|
||||||
|
url: 'https://popuporgies.com',
|
||||||
|
parent: 'aziani',
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
areaId: 8,
|
||||||
|
blockId: 116531,
|
||||||
|
seriesId: 395,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'azianiiron',
|
||||||
|
name: 'Aziani Iron',
|
||||||
|
url: 'https://aziani.com/azianiiron',
|
||||||
|
parent: 'aziani',
|
||||||
|
parameters: {
|
||||||
|
areaId: 3,
|
||||||
|
blockId: 114458,
|
||||||
|
seriesId: 105,
|
||||||
|
scene: 'https://aziani.com',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'mrsaltys',
|
||||||
|
name: 'Mr. Saltys',
|
||||||
|
url: 'https://aziani.com/mrsaltys',
|
||||||
|
parent: 'aziani',
|
||||||
|
hasLogo: false,
|
||||||
|
parameters: {
|
||||||
|
areaId: 3,
|
||||||
|
blockId: 114458,
|
||||||
|
seriesId: 106,
|
||||||
|
scene: 'https://aziani.com',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
/* offline
|
/* offline
|
||||||
{
|
|
||||||
slug: 'portagloryhole',
|
|
||||||
name: 'Porta Gloryhole',
|
|
||||||
url: 'https://www.portagloryhole.com',
|
|
||||||
parent: 'aziani',
|
|
||||||
tags: ['gloryhole'],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
slug: 'azianiiron',
|
|
||||||
name: 'Aziani Iron',
|
|
||||||
url: 'https://www.azianiiron.com',
|
|
||||||
parent: 'aziani',
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
slug: 'azianixposed',
|
slug: 'azianixposed',
|
||||||
name: 'Aziani Xposed',
|
name: 'Aziani Xposed',
|
||||||
|
@ -914,6 +950,42 @@ const sites = [
|
||||||
parent: 'aziani',
|
parent: 'aziani',
|
||||||
},
|
},
|
||||||
*/
|
*/
|
||||||
|
// AZIANI GAMMA
|
||||||
|
{
|
||||||
|
slug: 'gangbangcreampie',
|
||||||
|
name: 'Gangbang Creampie',
|
||||||
|
url: 'https://www.gangbangcreampie.com',
|
||||||
|
parent: 'gamma',
|
||||||
|
tags: ['gangbang', 'creampie'],
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
scene: 'https://www.gangbangcreampie.com/en/video/gangbangcreampie',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'gloryholesecrets',
|
||||||
|
name: 'Glory Hole Secrets',
|
||||||
|
url: 'https://www.gloryholesecrets.com',
|
||||||
|
parent: 'gamma',
|
||||||
|
tags: ['gloryhole'],
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
scene: 'https://www.gloryholesecrets.com/en/video/gloryholesecrets',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
/* different layout
|
||||||
|
{
|
||||||
|
slug: 'portagloryhole',
|
||||||
|
name: 'Porta Gloryhole',
|
||||||
|
url: 'https://www.portagloryhole.com',
|
||||||
|
parent: 'gamma',
|
||||||
|
tags: ['gloryhole'],
|
||||||
|
independent: true,
|
||||||
|
parameters: {
|
||||||
|
scene: 'https://www.portagloryhole.com/scenes',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
*/
|
||||||
// BABES
|
// BABES
|
||||||
{
|
{
|
||||||
name: 'Babes',
|
name: 'Babes',
|
||||||
|
|
|
@ -168,7 +168,7 @@ function toBaseActors(actorsOrNames, release) {
|
||||||
|
|
||||||
// using top level parent widens the scope too much, e.g. different Gamma sites may not use the same actor database
|
// using top level parent widens the scope too much, e.g. different Gamma sites may not use the same actor database
|
||||||
// const entity = getRecursiveParent(release?.entity);
|
// const entity = getRecursiveParent(release?.entity);
|
||||||
const entity = (release?.entity?.indepdendent && release?.entity)
|
const entity = (release?.entity?.independent && release?.entity)
|
||||||
|| release?.entity?.parent
|
|| release?.entity?.parent
|
||||||
|| release?.entity
|
|| release?.entity
|
||||||
|| null;
|
|| null;
|
||||||
|
@ -308,7 +308,7 @@ function curateProfileEntry(profile) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const curatedProfileEntry = {
|
const curatedProfileEntry = {
|
||||||
...(profile.update !== false && { id: profile.update }),
|
...(typeof profile.update === 'number' && { id: profile.update }),
|
||||||
actor_id: profile.actorId,
|
actor_id: profile.actorId,
|
||||||
entity_id: profile.entity?.id || null,
|
entity_id: profile.entity?.id || null,
|
||||||
date_of_birth: profile.dateOfBirth,
|
date_of_birth: profile.dateOfBirth,
|
||||||
|
@ -552,21 +552,34 @@ async function upsertProfiles(profiles) {
|
||||||
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
const newProfileEntries = profiles.filter((profile) => !profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||||
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
const updatingProfileEntries = profiles.filter((profile) => profile.update).map((profile) => curateProfileEntry(profile)).filter(Boolean);
|
||||||
|
|
||||||
const newProfiles = await insertProfiles(newProfileEntries);
|
const newProfiles = newProfileEntries.length > 0
|
||||||
|
? await insertProfiles(newProfileEntries)
|
||||||
|
: [];
|
||||||
|
|
||||||
if (argv.force && updatingProfileEntries.length > 0) {
|
if (argv.force && updatingProfileEntries.length > 0) {
|
||||||
const transaction = await knex.transaction();
|
const transaction = await knex.transaction();
|
||||||
const queries = updatingProfileEntries.map((profileEntry) => knex('actors_profiles')
|
const queries = updatingProfileEntries.map(async (profileEntry) => knex('actors_profiles')
|
||||||
.where('id', profileEntry.id)
|
.modify((builder) => {
|
||||||
|
if (profileEntry.id) {
|
||||||
|
builder.where('id', profileEntry.id);
|
||||||
|
} else {
|
||||||
|
builder
|
||||||
|
.where('actor_id', profileEntry.actor_id)
|
||||||
|
.where('entity_id', profileEntry.entity_id);
|
||||||
|
}
|
||||||
|
})
|
||||||
.update(profileEntry)
|
.update(profileEntry)
|
||||||
.returning(['id', 'actor_id'])
|
.returning(['id', 'actor_id'])
|
||||||
.transacting(transaction));
|
.transacting(transaction));
|
||||||
|
|
||||||
await Promise.all(queries)
|
await Promise.all(queries)
|
||||||
.then(transaction.commit)
|
.then(transaction.commit)
|
||||||
.catch(transaction.rollback);
|
.catch((error) => {
|
||||||
|
logger.error(error.message);
|
||||||
|
return transaction.rollback();
|
||||||
|
});
|
||||||
|
|
||||||
logger.info(`Updated ${updatingProfileEntries.length} new actor profiles`);
|
logger.info(`Updated ${updatingProfileEntries.length} actor profiles`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (profiles.length > 0) {
|
if (profiles.length > 0) {
|
||||||
|
@ -586,10 +599,12 @@ async function upsertProfiles(profiles) {
|
||||||
media_id: profile.avatarMediaId,
|
media_id: profile.avatarMediaId,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await knex('actors_avatars')
|
if (avatars.length > 0) {
|
||||||
.insert(avatars)
|
await knex('actors_avatars')
|
||||||
.onConflict()
|
.insert(avatars)
|
||||||
.ignore();
|
.onConflict()
|
||||||
|
.ignore();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -599,7 +614,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
||||||
const profiles = Promise.map(validSources, async (source) => {
|
const profiles = Promise.map(validSources, async (source) => {
|
||||||
try {
|
try {
|
||||||
// config may group sources to try until success
|
// config may group sources to try until success
|
||||||
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
return [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
||||||
try {
|
try {
|
||||||
const entity = entitiesBySlug[scraperSlug] || null;
|
const entity = entitiesBySlug[scraperSlug] || null;
|
||||||
|
|
||||||
|
@ -846,6 +861,13 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||||
OR actors.entry_id = base_actors.entry_id)
|
OR actors.entry_id = base_actors.entry_id)
|
||||||
`);
|
`);
|
||||||
|
|
||||||
|
const actorIds = existingActors.map((actor) => actor.id);
|
||||||
|
const entityIds = Array.from(new Set(baseActors.map((actor) => actor.entity?.id).filter(Boolean)));
|
||||||
|
|
||||||
|
const existingProfiles = await knex('actors_profiles')
|
||||||
|
.whereIn('actor_id', actorIds)
|
||||||
|
.whereIn('entity_id', entityIds);
|
||||||
|
|
||||||
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
|
// const existingActorSlugs = new Set(existingActors.map(actor => actor.slug));
|
||||||
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
|
const existingActorSlugs = existingActors.reduce((acc, actor) => ({
|
||||||
...acc,
|
...acc,
|
||||||
|
@ -863,7 +885,7 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||||
|
|
||||||
const newActors = await bulkInsert('actors', curatedActorEntries);
|
const newActors = await bulkInsert('actors', curatedActorEntries);
|
||||||
|
|
||||||
const newActorIdsByEntityIdEntryIdAndSlug = newActors.reduce((acc, actor) => ({
|
const actorIdsByEntityIdEntryIdAndSlug = [...existingActors, ...newActors].reduce((acc, actor) => ({
|
||||||
...acc,
|
...acc,
|
||||||
[actor.entity_id]: {
|
[actor.entity_id]: {
|
||||||
...acc[actor.entity_id],
|
...acc[actor.entity_id],
|
||||||
|
@ -874,13 +896,26 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||||
},
|
},
|
||||||
}), {});
|
}), {});
|
||||||
|
|
||||||
|
const profileIdsByActorIdAndEntityId = existingProfiles.reduce((acc, profile) => ({
|
||||||
|
...acc,
|
||||||
|
[profile.actor_id]: {
|
||||||
|
...acc[profile.actor_id],
|
||||||
|
[profile.entity_id]: profile.id,
|
||||||
|
},
|
||||||
|
}), {});
|
||||||
|
|
||||||
const newActorProfiles = await Promise.all(baseActors
|
const newActorProfiles = await Promise.all(baseActors
|
||||||
.filter((actor) => actor.hasProfile)
|
.filter((actor) => actor.hasProfile)
|
||||||
.map((actor) => ({
|
.map((actor) => {
|
||||||
...actor,
|
const actorId = actorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || actorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug];
|
||||||
actorId: newActorIdsByEntityIdEntryIdAndSlug[actor.entity?.id]?.[actor.entryId]?.[actor.slug] || newActorIdsByEntityIdEntryIdAndSlug.null?.null?.[actor.slug],
|
|
||||||
}))
|
return {
|
||||||
.filter((actor) => !!actor.id)
|
...actor,
|
||||||
|
actorId,
|
||||||
|
update: profileIdsByActorIdAndEntityId[actorId]?.[actor.entity?.id],
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((actor) => !!actor.actorId)
|
||||||
.map((actor) => curateProfile(actor)));
|
.map((actor) => curateProfile(actor)));
|
||||||
|
|
||||||
await storeProfiles(newActorProfiles);
|
await storeProfiles(newActorProfiles);
|
||||||
|
|
|
@ -1,147 +1,187 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
|
const unprint = require('unprint');
|
||||||
|
const { decode } = require('html-entities');
|
||||||
|
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const { get, getAll, initAll, extractDate } = require('../utils/qu');
|
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
|
||||||
const { feetInchesToCm } = require('../utils/convert');
|
|
||||||
|
|
||||||
const imageRegex = /-\dx.jpg/;
|
function scrapeScene(data, channel, parameters) {
|
||||||
|
const release = {};
|
||||||
|
|
||||||
function getFallbacks(source) {
|
release.entryId = data.cms_set_id;
|
||||||
if (!source || source.includes('join.jpg')) {
|
release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(new Set([
|
release.title = data.name;
|
||||||
source.replace(imageRegex, '-4x.jpg'),
|
release.description = data.description;
|
||||||
source.replace(imageRegex, '-3x.jpg'),
|
|
||||||
source.replace(imageRegex, '-2x.jpg'),
|
|
||||||
source.replace(imageRegex, '-1x.jpg'),
|
|
||||||
source,
|
|
||||||
]));
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeAll(scenes, site) {
|
release.date = unprint.extractDate(data.added_nice, 'YYYY-MM-DD');
|
||||||
return scenes.map(({ qu }) => {
|
release.duration = Number(data.lengths.total);
|
||||||
const release = {};
|
|
||||||
|
|
||||||
release.url = qu.url('a');
|
release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models')?.data_values.map((actor) => ({
|
||||||
|
name: actor.name,
|
||||||
release.title = qu.q('h5 a', true);
|
url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
|
||||||
release.date = qu.date('.icon-calendar + strong', 'MM/DD/YYYY');
|
|
||||||
|
|
||||||
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(release.url).pathname.match(/trailers\/(.*).html/)?.[1];
|
|
||||||
|
|
||||||
release.actors = qu.q('h3', true).replace(/featuring:\s?/i, '').split(', ').filter(Boolean);
|
|
||||||
|
|
||||||
const photoCount = qu.q('.stdimage', 'cnt');
|
|
||||||
|
|
||||||
[release.poster, ...release.photos] = Array.from({ length: Number(photoCount) }, (value, index) => {
|
|
||||||
const source = qu.img('.stdimage', `src${index}_1x`, { origin: site.url });
|
|
||||||
|
|
||||||
return getFallbacks(source);
|
|
||||||
});
|
|
||||||
|
|
||||||
return release;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrapeScene({ html, qu }, url, channel) {
|
|
||||||
const release = { url };
|
|
||||||
|
|
||||||
release.entryId = qu.q('.stdimage', 'id', true)?.match(/set-target-(\d+)/)?.[1] || new URL(url).pathname.match(/trailers\/(.*).html/)?.[1];
|
|
||||||
|
|
||||||
release.title = qu.q('h2', true);
|
|
||||||
release.description = qu.q('p', true);
|
|
||||||
|
|
||||||
release.date = extractDate(html, 'MM/DD/YYYY', /\b\d{2}\/\d{2}\/\d{4}\b/);
|
|
||||||
|
|
||||||
release.actors = qu.all('h5:not(.video_categories) a').map((actor) => ({
|
|
||||||
name: qu.q(actor, null, true),
|
|
||||||
url: qu.url(actor, null),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
release.tags = qu.all('.video_categories a', true);
|
release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
|
||||||
|
name: director.name,
|
||||||
|
// url: `${channel.url}/model/${director.slug}?models=${director.name}`,
|
||||||
|
}));
|
||||||
|
|
||||||
release.duration = qu.dur('.video_categories + p');
|
release.tags = data.data_types
|
||||||
|
.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
|
||||||
|
.flatMap((tags) => tags.data_values.map((tag) => tag.name));
|
||||||
|
|
||||||
release.poster = getFallbacks(qu.img('a img')) || getFallbacks(qu.img('#preview video', 'poster', { origin: channel.url }));
|
const poster = data.preview_formatted.thumb;
|
||||||
release.caps = qu.imgs('.featured-video img', 'src0_1x', { origin: channel.url }).map((source) => getFallbacks(source)).filter(Boolean);
|
const teaser = data.preview_formatted.clip?.[0];
|
||||||
|
const trailer = data.preview_formatted.trailer?.formats?.[0]?.content?.[0];
|
||||||
|
|
||||||
release.trailer = qu.video('#preview source');
|
if (poster) {
|
||||||
|
release.poster = Object.keys(poster)
|
||||||
|
.filter((key) => poster[key].length > 0)
|
||||||
|
.toSorted((keyA, keyB) => keyB.split('-')[0] - keyA.split('-')[0])
|
||||||
|
.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (teaser && teaser.fileuri !== trailer?.fileuri) {
|
||||||
|
release.teaser = unprint.prefixUrl(`${teaser.fileuri}?${teaser.signature}`, parameters.cdn);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trailer) {
|
||||||
|
release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
|
||||||
|
}
|
||||||
|
|
||||||
|
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeProfile({ el, qu }) {
|
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||||
const profile = {};
|
const query = new URLSearchParams({
|
||||||
|
cms_area_id: parameters.areaId,
|
||||||
|
cms_block_id: parameters.blockId,
|
||||||
|
count: 100,
|
||||||
|
start: (page - 1) * 100,
|
||||||
|
orderby: 'published_desc',
|
||||||
|
content_type: 'video',
|
||||||
|
// unsure what this does
|
||||||
|
status: 'enabled',
|
||||||
|
cms_set_ids: undefined,
|
||||||
|
data_types: 1,
|
||||||
|
content_count: 1,
|
||||||
|
data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
|
||||||
|
}).toString();
|
||||||
|
|
||||||
const bio = Array.from(qu.q('.widget-content').childNodes).reduce((acc, node, index, nodes) => {
|
const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||||
const nextNode = nodes[index + 1];
|
|
||||||
|
|
||||||
if (node.tagName === 'STRONG' && nextNode?.nodeType === 3) {
|
const res = await unprint.get(url, {
|
||||||
acc[slugify(node.textContent, '_')] = nextNode.textContent.trim();
|
headers: {
|
||||||
}
|
Referer: channel.url,
|
||||||
|
'x-nats-cms-area-id': parameters.areaId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
return acc;
|
if (res.ok && res.data.success) {
|
||||||
}, {});
|
return res.data.sets.map((data) => scrapeScene(data, channel, parameters));
|
||||||
|
|
||||||
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
|
|
||||||
if (bio.age) profile.age = Number(bio.age);
|
|
||||||
|
|
||||||
if (bio.height && /\d{3}/.test(bio.height)) profile.height = Number(bio.height.match(/\d+/)[0]);
|
|
||||||
if (bio.height && /\d[;']\d/.test(bio.height)) profile.height = feetInchesToCm(bio.height);
|
|
||||||
|
|
||||||
if (bio.measurements) {
|
|
||||||
const [bust, waist, hip] = bio.measurements.split('-');
|
|
||||||
|
|
||||||
if (bust && /\d+[a-zA-Z]+/.test(bust)) profile.bust = bust;
|
|
||||||
if (waist) profile.waist = Number(waist);
|
|
||||||
if (hip) profile.hip = Number(hip);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bio.bust_size && !profile.bust) profile.bust = bio.bust_size.toUpperCase();
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
if (bio.birth_location) profile.birthPlace = bio.birth_location;
|
async function fetchScene(url, entity, _baseRelease, { parameters }) {
|
||||||
if (bio.status_married_or_single) profile.relationship = bio.status_married_or_single;
|
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
|
||||||
|
|
||||||
if (bio.eye_color) profile.eyes = bio.eye_color;
|
const query = new URLSearchParams({
|
||||||
|
cms_set_ids: entryId,
|
||||||
|
cms_area_id: parameters.areaId,
|
||||||
|
cms_block_id: parameters.blockId,
|
||||||
|
content: 1,
|
||||||
|
orderby: 'published_desc',
|
||||||
|
content_type: 'video',
|
||||||
|
// unsure what this does
|
||||||
|
data_types: 1,
|
||||||
|
content_count: 1,
|
||||||
|
}).toString();
|
||||||
|
|
||||||
const avatar = qu.img('.tac img');
|
const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||||
profile.avatar = getFallbacks(avatar);
|
|
||||||
|
|
||||||
profile.releases = scrapeAll(initAll(el, '.featured-video'));
|
const res = await unprint.get(apiUrl, {
|
||||||
|
headers: {
|
||||||
|
Referer: entity.url,
|
||||||
|
'x-nats-cms-area-id': parameters.areaId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res.ok && res.data.success) {
|
||||||
|
return scrapeScene(res.data.sets[0], entity, parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeProfile(data, entity, parameters) {
|
||||||
|
const profile = {};
|
||||||
|
|
||||||
|
const bio = Object.fromEntries(Object.values(data.data_detail_values).map((detail) => [
|
||||||
|
slugify(detail.name, '_'),
|
||||||
|
decode(detail.value || detail.content_formatted || detail.content),
|
||||||
|
]));
|
||||||
|
|
||||||
|
profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
|
||||||
|
profile.entryId = data.cms_data_value_id;
|
||||||
|
|
||||||
|
profile.description = data.description;
|
||||||
|
|
||||||
|
profile.gender = bio.gender?.toLowerCase();
|
||||||
|
profile.age = bio.age;
|
||||||
|
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
|
||||||
|
|
||||||
|
profile.measurements = bio.measurements;
|
||||||
|
profile.height = feetInchesToCm(bio.height);
|
||||||
|
profile.foot = femaleFeetUsToEu(bio.foot_size);
|
||||||
|
|
||||||
|
profile.hairColor = bio.hair_color;
|
||||||
|
profile.eyeColor = bio.eye_color;
|
||||||
|
|
||||||
|
const avatar = bio.thumbnail?.image[0];
|
||||||
|
|
||||||
|
if (avatar) {
|
||||||
|
profile.avatar = {
|
||||||
|
src: `${parameters.cdn}${avatar.fileuri}?${avatar.signature}`,
|
||||||
|
expectType: {
|
||||||
|
'application/octet-stream': 'image/jpeg',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(site, page) {
|
async function fetchProfile({ url }, { entity, parameters }) {
|
||||||
const url = `${site.url}/tour/categories/movies_${page}_d.html`;
|
if (!url) {
|
||||||
const res = await getAll(url, '.featured-video');
|
// no easy search option
|
||||||
|
return null;
|
||||||
if (res.ok) {
|
|
||||||
return scrapeAll(res.items, site);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.status;
|
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchScene(url, site) {
|
const query = new URLSearchParams({
|
||||||
const res = await get(url, '.trailer');
|
cms_data_value_ids: actorId,
|
||||||
|
cms_block_id: entity.parameters.modelBlockId,
|
||||||
|
cms_data_type_id: 4,
|
||||||
|
}).toString();
|
||||||
|
|
||||||
if (res.ok) {
|
const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
|
||||||
return scrapeScene(res.item, url, site);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.status;
|
const res = await unprint.get(apiUrl, {
|
||||||
}
|
headers: {
|
||||||
|
Referer: entity.url,
|
||||||
|
'x-nats-cms-area-id': entity.parameters.areaId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
async function fetchProfile({ name: actorName }, { site }) {
|
if (res.ok && res.data.success) {
|
||||||
const actorSlug = slugify(actorName, '');
|
return scrapeProfile(res.data.data_values[0], entity, parameters);
|
||||||
const url = `${site.url}/tour/models/${actorSlug}.html`;
|
|
||||||
const res = await get(url, '.page-content .row');
|
|
||||||
|
|
||||||
if (res.ok) {
|
|
||||||
return scrapeProfile(res.item);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.status;
|
return res.status;
|
||||||
|
|
|
@ -7,6 +7,7 @@ const assylum = require('./assylum');
|
||||||
const amateurallure = require('./amateurallure');
|
const amateurallure = require('./amateurallure');
|
||||||
const americanpornstar = require('./americanpornstar');
|
const americanpornstar = require('./americanpornstar');
|
||||||
const amnesiac = require('./amnesiac');
|
const amnesiac = require('./amnesiac');
|
||||||
|
const aziani = require('./aziani');
|
||||||
const badoink = require('./badoink');
|
const badoink = require('./badoink');
|
||||||
const bamvisions = require('./bamvisions');
|
const bamvisions = require('./bamvisions');
|
||||||
const bang = require('./bang');
|
const bang = require('./bang');
|
||||||
|
@ -94,6 +95,7 @@ const scrapers = {
|
||||||
archangel,
|
archangel,
|
||||||
asiam: modelmedia,
|
asiam: modelmedia,
|
||||||
assylum,
|
assylum,
|
||||||
|
aziani,
|
||||||
badoink,
|
badoink,
|
||||||
bamvisions,
|
bamvisions,
|
||||||
bang,
|
bang,
|
||||||
|
@ -201,6 +203,9 @@ const scrapers = {
|
||||||
anilos: nubiles,
|
anilos: nubiles,
|
||||||
archangel,
|
archangel,
|
||||||
asiam: modelmedia,
|
asiam: modelmedia,
|
||||||
|
aziani,
|
||||||
|
'2poles1hole': aziani,
|
||||||
|
creampiled: aziani,
|
||||||
babes: aylo,
|
babes: aylo,
|
||||||
babevr: badoink,
|
babevr: badoink,
|
||||||
baddaddypov: fullpornnetwork,
|
baddaddypov: fullpornnetwork,
|
||||||
|
|
|
@ -87,15 +87,24 @@ function convertApi(input, fromOrTo, to) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function maleFeetUsToEu(input) {
|
function maleFeetUsToEu(input) {
|
||||||
|
if (!input) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
||||||
|
|
||||||
return Math.round((1.27 * size + 29.94) / 0.5) * 0.5; // round to nearest half
|
return Math.round((1.27 * size + 29.94) / 0.5) * 0.5; // round to nearest half
|
||||||
}
|
}
|
||||||
|
|
||||||
function femaleFeetUsToEu(input) {
|
function femaleFeetUsToEu(input) {
|
||||||
|
if (!input) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
const size = Number(input.toString().match(/\d+(\.\d+)?/)?.[0]);
|
||||||
|
|
||||||
return Math.round((1.27 * size + 28.67) / 0.5) * 0.5; // round to nearest half
|
// return Math.round((1.27 * size + 28.67) / 0.5) * 0.5; // round to nearest half
|
||||||
|
return Math.round(1.27 * size + 28.67);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
Loading…
Reference in New Issue