DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Refreshing entity slug cache in seeds. Added Hardwerk to Radical.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-03-15 17:42:07 +01:00
parent 0560fac1ff
commit 0d30115ad5
6 changed files with 183 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
seeds/01_networks.js
View File

@@ -2,6 +2,14 @@
const omit = require('object.omit'); const omit = require('object.omit');
const upsert = require('../src/utils/upsert'); const upsert = require('../src/utils/upsert');
const redis = require('../src/redis');
const entityPrefixes = {
channel: '',
network: '_',
studio: '*',
info: '@',
};
const grandParentNetworks = [ const grandParentNetworks = [
{ {
@@ -905,80 +913,88 @@ const networks = [
}, },
]; ];
exports.seed = (knex) => Promise.resolve() exports.seed = async (knex) => {
.then(async () => { await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => {
await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => { if (network.rename) {
if (network.rename) { return knex('entities')
return knex('entities') .where({
.where({ type: network.type || 'network',
type: network.type || 'network', slug: network.rename,
slug: network.rename, })
}) .update('slug', network.slug);
.update('slug', network.slug); }
}
return null; return null;
}).filter(Boolean)); }).filter(Boolean));
const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex); const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex);
const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const parentNetworksWithGrandParent = parentNetworks.map((network) => ({ const parentNetworksWithGrandParent = parentNetworks.map((network) => ({
slug: network.slug, slug: network.slug,
name: network.name, name: network.name,
type: network.type || 'network', type: network.type || 'network',
alias: network.alias, alias: network.alias,
url: network.url, url: network.url,
description: network.description, description: network.description,
has_logo: network.hasLogo ?? true, has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true, showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null, parameters: network.parameters || null,
options: network.options, options: network.options,
parent_id: grandParentNetworksBySlug[network.parent] || null, parent_id: grandParentNetworksBySlug[network.parent] || null,
})); }));
const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex); const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex);
const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const networksWithParent = networks.map((network) => ({ const networksWithParent = networks.map((network) => ({
slug: network.slug, slug: network.slug,
name: network.name, name: network.name,
type: network.type || 'network', type: network.type || 'network',
alias: network.alias, alias: network.alias,
url: network.url, url: network.url,
description: network.description, description: network.description,
has_logo: network.hasLogo ?? true, has_logo: network.hasLogo ?? true,
showcased: typeof network.showcased === 'boolean' ? network.showcased : true, showcased: typeof network.showcased === 'boolean' ? network.showcased : true,
parameters: network.parameters || null, parameters: network.parameters || null,
options: network.options, options: network.options,
parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null, parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null,
})); }));
const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex); const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex);
const networkIdsBySlug = [].concat( const networkIdsBySlug = [].concat(
grandParentNetworkEntries.inserted, grandParentNetworkEntries.inserted,
grandParentNetworkEntries.updated, grandParentNetworkEntries.updated,
parentNetworkEntries.inserted, parentNetworkEntries.inserted,
parentNetworkEntries.updated, parentNetworkEntries.updated,
networkEntries.inserted, networkEntries.inserted,
networkEntries.updated, networkEntries.updated,
).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); ).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean); const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean);
const tagEntries = await knex('tags').whereIn('slug', tagSlugs); const tagEntries = await knex('tags').whereIn('slug', tagSlugs);
const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {}); const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {});
const tagAssociations = networks const tagAssociations = networks
.map((network) => (network.tags .map((network) => (network.tags
? network.tags.map((tagSlug) => ({ ? network.tags.map((tagSlug) => ({
entity_id: networkIdsBySlug[network.slug], entity_id: networkIdsBySlug[network.slug],
tag_id: tagIdsBySlug[tagSlug], tag_id: tagIdsBySlug[tagSlug],
inherit: true, inherit: true,
})) }))
: [])) : []))
.flat(); .flat();
await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex); await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
});
const entities = await knex('entities').select('id', 'slug', 'type');
await redis.connect();
await redis.del('traxxx:entities:id_by_slug');
await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
await redis.disconnect();
};

275
seeds/02_sites.js
View File

@@ -1,4 +1,12 @@
const upsert = require('../src/utils/upsert'); const upsert = require('../src/utils/upsert');
const redis = require('../src/redis');
const entityPrefixes = {
channel: '',
network: '_',
studio: '*',
info: '@',
};
/* eslint-disable max-len */ /* eslint-disable max-len */
const sites = [ const sites = [
@@ -10554,6 +10562,18 @@ const sites = [
siteAsSerie: true, siteAsSerie: true,
}, },
}, },
{
name: 'Hardwerk',
slug: 'hardwerk',
url: 'https://hardwerk.com',
independent: true,
parent: 'radical',
parameters: {
endpoint: 'jC4SrjH8YVDtRejiA0PMx',
videos: 'films',
actors: 'performers',
},
},
// REALITY KINGS // REALITY KINGS
{ {
name: 'Look At Her Now', name: 'Look At Her Now',
@@ -15540,195 +15560,86 @@ sites.reduce((acc, site) => {
}, new Set()); }, new Set());
/* eslint-disable max-len */ /* eslint-disable max-len */
exports.seed = (knex) => Promise.resolve() exports.seed = async (knex) => {
.then(async () => { await Promise.all(sites.map(async (channel) => {
await Promise.all(sites.map(async (channel) => { if (channel.rename) {
if (channel.rename) { await knex('entities')
await knex('entities') .where({
.where({ type: channel.type || 'channel',
type: channel.type || 'channel', slug: channel.rename,
slug: channel.rename, })
}) .update('slug', channel.slug);
.update('slug', channel.slug);
return; return;
}
if (channel.delete) {
await knex('entities')
.where({
type: channel.type || 'channel',
slug: channel.slug,
})
.delete();
}
}).filter(Boolean));
const networks = await knex('entities')
.where('type', 'network')
.orWhereNull('parent_id');
const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tags = await knex('tags').select('*').whereNull('alias_for');
const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({
slug: site.slug,
name: site.name,
name_stylized: site.style,
type: site.type || 'channel',
alias: site.alias,
description: site.description,
url: site.url,
parameters: site.parameters || null,
options: site.options,
parent_id: networksMap[site.parent] || null,
priority: site.priority || 0,
independent: !!site.independent,
visible: site.visible,
showcased: site.showcased,
has_logo: site.hasLogo === undefined ? true : site.hasLogo,
}));
const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex);
const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tagAssociations = sites.map((site) => (site.tags && !site.delete
? site.tags.map((tagSlug) => {
const tag = tagsMap[tagSlug];
if (!tag) {
console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
} }
if (channel.delete) { return {
await knex('entities') entity_id: sitesMap[site.slug],
.where({ tag_id: tagsMap[tagSlug],
type: channel.type || 'channel', inherit: true,
slug: channel.slug, };
}) })
.delete(); : []
} )).flat();
}).filter(Boolean));
const networks = await knex('entities') await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
.where('type', 'network')
.orWhereNull('parent_id');
const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); const entities = await knex('entities').select('id', 'slug', 'type');
const tags = await knex('tags').select('*').whereNull('alias_for'); await redis.connect();
const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({ await redis.del('traxxx:entities:id_by_slug');
slug: site.slug, await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id]));
name: site.name,
name_stylized: site.style,
type: site.type || 'channel',
alias: site.alias,
description: site.description,
url: site.url,
parameters: site.parameters || null,
options: site.options,
parent_id: networksMap[site.parent] || null,
priority: site.priority || 0,
independent: !!site.independent,
visible: site.visible,
showcased: site.showcased,
has_logo: site.hasLogo === undefined ? true : site.hasLogo,
}));
const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex); await redis.disconnect();
const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); };
const tagAssociations = sites.map((site) => (site.tags && !site.delete
? site.tags.map((tagSlug) => {
const tag = tagsMap[tagSlug];
if (!tag) {
console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`);
}
return {
entity_id: sitesMap[site.slug],
tag_id: tagsMap[tagSlug],
inherit: true,
};
})
: []
)).flat();
return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
});
exports.sites = sites; exports.sites = sites;
/*
'X-Art' => 'xart',
'met-art' => 'metart',
'18og' => '18OnlyGirls',
'a1o1' => 'Asian1on1',
'add' => 'ManualAddActors',
'analb' => 'AnalBeauty',
'bgonzo' => 'BangGonzo',
'btlbd' => 'BigTitsLikeBigDicks',
'bjf' => 'BlowjobFridays',
'cws' => 'CzechWifeSwap',
'Daughter' => 'DaughterSwap',
'Daughters' => 'DaughterSwap',
'dc' => 'DorcelVision',
'dpg' => 'DigitalPlayground',
'dsw' => 'DaughterSwap',
'faq' => 'FirstAnalQuest',
'ft' => 'FastTimes',
'fittingroom' => 'Fitting-Room',
'gbcp' => 'GangbangCreampie',
'hart' => 'Hegre',
'hegre-art' => 'Hegre',
'kha' => 'KarupsHA',
'kow' => 'KarupsOW',
'kpc' => 'KarupsPC',
'la' => 'LatinAdultery',
'lcd' => 'LittleCaprice',
'lhf' => 'LoveHerFeet',
'littlecapricedreams' => 'Little Caprice Dreams',
'maj' => 'ManoJob',
'mfl' => 'Mofos',
'mj' => 'ManoJob',
'mpov' => 'MrPOV',
'naughtyamericavr' => 'NaughtyAmerica',
'news' => 'NewSensations',
'ps' => 'PropertySex',
'sart' => 'SexArt',
'sbj' => 'StreetBlowjobs',
'sislove' => 'SisLovesMe',
'tds' => 'TheDickSuckers',
'these' => 'TheStripperExperience',
'tlc' => 'TeensLoveCream',
'tle' => 'TheLifeErotic',
'tog' => 'TonightsGirlfriend',
'wowg' => 'WowGirls',
'wy' => 'WebYoung',
'itc' => 'InTheCrack',
"abbw" => "AbbyWinters",
"abme" => "AbuseMe",
"ana" => "AnalAngels",
"atke" => "ATKExotics",
"atkg" => "ATKGalleria",
"atkgfs" => "ATKGirlfriends",
"atkh" => "ATKHairy",
"aktp" => "ATKPetites",
"ba" => "Beauty-Angels",
"bna" => "BrandNew",
"bam" => "BruceAndMorgan",
"bcast" => "BrutalCastings",
"bd" => "BrutalDildos",
"bpu" => "BrutalPickups",
"cza" => "CzhecAmateurs",
"czbb" => "CzechBangBus",
"czb" => "CzechBitch",
"cc" => "CzechCasting",
"czc" => "CzechCouples",
"czestro" => "CzechEstrogenolit",
"czf" => "CzechFantasy",
"czgb" => "CzechGangBang",
"cgfs" => "CzechGFS",
"czharem" => "CzechHarem",
"czm" => "CzechMassage",
"czo" => "CzechOrgasm",
"czps" => "CzechPawnShop",
"css" => "CzechStreets",
"cztaxi" => "CzechTaxi",
"czt" => "CzechTwins",
"dts" => "DeepThroatSirens",
"doan" => "DiaryOfANanny",
"ds" => "DungeonSex",
"ffr" => "FacialsForever",
"ff" => "FilthyFamily",
"fbbg" => "FirstBGG",
"fs" => "FuckStudies",
"tfcp" => "FullyClothedPissing",
"gdp" => "GirlsDoPorn",
"Harmony" => "HarmonyVision",
"hletee" => "HelplessTeens",
"jlmf" => "JessieLoadsMonsterFacials",
"lang" => "LANewGirl",
"mmp" => "MMPNetwork",
"mbc" => "MyBabysittersClub",
"nvg" => "NetVideoGirls",
"oo" => "Only-Opaques",
"os" => "Only-Secretaries",
"oss" => "OnlySilAndSatin",
"psus" => "PascalsSubSluts",
"psp" => "PorsntarsPunishment",
"pdmqfo" => "QuestForOrgasm",
"sed" => "SexualDisgrace",
"sislov" => "SisLovesMe",
"tslw" => "SlimeWave",
"stre" => "StrictRestraint",
"t18" => "Taboo18",
"tsma" => "TeenSexMania",
"tsm" => "TeenSexMovs",
"ttw" => "TeensInTheWoods",
"tgw" => "ThaiGirlsWild",
"taob" => "TheArtOfBlowJob",
"trwo" => "TheRealWorkout",
"tt" => "TryTeens",
"vp" => "VIPissy",
"wrh" => "WeAreHairy",
"yt" => "YoungThroats",
];
*/

5
seeds/06_affiliates.js
View File

@@ -719,6 +719,11 @@ const affiliates = [
url: 'https://register.join-toughlovex.com/track/MzAwMDA5NzkuMy43Ni4xOTcuMC4wLjAuMC4w', url: 'https://register.join-toughlovex.com/track/MzAwMDA5NzkuMy43Ni4xOTcuMC4wLjAuMC4w',
comment: 'rev share', comment: 'rev share',
}, },
{
channel: 'hardwerk',
url: 'https://register.hardwerk.com/track/MzAwMDA5NzkuMy4xNTEuMzM5LjAuMC4wLjAuMA',
comment: 'rev share',
},
// radical > topwebmodels // radical > topwebmodels
{ {
network: 'topwebmodels', network: 'topwebmodels',

1
src/scrapers/actors.js
View File

@@ -140,6 +140,7 @@ module.exports = {
purgatoryx: radical, purgatoryx: radical,
topwebmodels: radical, topwebmodels: radical,
lucidflix: radical, lucidflix: radical,
hardwerk: radical,
// hush / hussiepass // hush / hussiepass
eyeontheguy: hush, eyeontheguy: hush,
hushpass: hush, hushpass: hush,

2
src/scrapers/radical.js
View File

@@ -215,7 +215,7 @@ function scrapeProfile(data, channel, scenes, parameters) {
async function fetchProfile(actor, { channel, parameters }) { async function fetchProfile(actor, { channel, parameters }) {
const endpoint = await fetchEndpoint(channel); const endpoint = await fetchEndpoint(channel);
const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`); const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
if (res.ok && res.body.pageProps?.model) { if (res.ok && res.body.pageProps?.model) {
return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters); return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);

1
tests/profiles.js
View File

@@ -153,6 +153,7 @@ const actors = [
{ entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] },
{ entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] }, { entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] },
{ entity: 'hardwerk', name: 'Luna Silver', fields: ['avatar', 'gender'] },
// wankz // wankz
{ entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] }, { entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] },
{ entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] }, { entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] },