diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 12d023cd..4c90d5d2 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -2,6 +2,14 @@ const omit = require('object.omit'); const upsert = require('../src/utils/upsert'); +const redis = require('../src/redis'); + +const entityPrefixes = { + channel: '', + network: '_', + studio: '*', + info: '@', +}; const grandParentNetworks = [ { @@ -905,80 +913,88 @@ const networks = [ }, ]; -exports.seed = (knex) => Promise.resolve() - .then(async () => { - await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => { - if (network.rename) { - return knex('entities') - .where({ - type: network.type || 'network', - slug: network.rename, - }) - .update('slug', network.slug); - } +exports.seed = async (knex) => { + await Promise.all([].concat(grandParentNetworks, parentNetworks, networks).map(async (network) => { + if (network.rename) { + return knex('entities') + .where({ + type: network.type || 'network', + slug: network.rename, + }) + .update('slug', network.slug); + } - return null; - }).filter(Boolean)); + return null; + }).filter(Boolean)); - const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex); - const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); + const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => (omit({ ...network, type: 'network' }, 'rename'))), ['slug', 'type'], knex); + const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const parentNetworksWithGrandParent = parentNetworks.map((network) => ({ - slug: network.slug, - name: network.name, - type: network.type || 'network', - alias: network.alias, - url: network.url, - description: network.description, - has_logo: network.hasLogo ?? true, - showcased: typeof network.showcased === 'boolean' ? network.showcased : true, - parameters: network.parameters || null, - options: network.options, - parent_id: grandParentNetworksBySlug[network.parent] || null, - })); + const parentNetworksWithGrandParent = parentNetworks.map((network) => ({ + slug: network.slug, + name: network.name, + type: network.type || 'network', + alias: network.alias, + url: network.url, + description: network.description, + has_logo: network.hasLogo ?? true, + showcased: typeof network.showcased === 'boolean' ? network.showcased : true, + parameters: network.parameters || null, + options: network.options, + parent_id: grandParentNetworksBySlug[network.parent] || null, + })); - const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex); - const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); + const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex); + const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const networksWithParent = networks.map((network) => ({ - slug: network.slug, - name: network.name, - type: network.type || 'network', - alias: network.alias, - url: network.url, - description: network.description, - has_logo: network.hasLogo ?? true, - showcased: typeof network.showcased === 'boolean' ? network.showcased : true, - parameters: network.parameters || null, - options: network.options, - parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null, - })); + const networksWithParent = networks.map((network) => ({ + slug: network.slug, + name: network.name, + type: network.type || 'network', + alias: network.alias, + url: network.url, + description: network.description, + has_logo: network.hasLogo ?? true, + showcased: typeof network.showcased === 'boolean' ? network.showcased : true, + parameters: network.parameters || null, + options: network.options, + parent_id: parentNetworksBySlug[network.parent] || grandParentNetworksBySlug[network.parent] || null, + })); - const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex); + const networkEntries = await upsert('entities', networksWithParent, ['slug', 'type'], knex); - const networkIdsBySlug = [].concat( - grandParentNetworkEntries.inserted, - grandParentNetworkEntries.updated, - parentNetworkEntries.inserted, - parentNetworkEntries.updated, - networkEntries.inserted, - networkEntries.updated, - ).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); + const networkIdsBySlug = [].concat( + grandParentNetworkEntries.inserted, + grandParentNetworkEntries.updated, + parentNetworkEntries.inserted, + parentNetworkEntries.updated, + networkEntries.inserted, + networkEntries.updated, + ).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean); + const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean); - const tagEntries = await knex('tags').whereIn('slug', tagSlugs); - const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {}); + const tagEntries = await knex('tags').whereIn('slug', tagSlugs); + const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {}); - const tagAssociations = networks - .map((network) => (network.tags - ? network.tags.map((tagSlug) => ({ - entity_id: networkIdsBySlug[network.slug], - tag_id: tagIdsBySlug[tagSlug], - inherit: true, - })) - : [])) - .flat(); + const tagAssociations = networks + .map((network) => (network.tags + ? network.tags.map((tagSlug) => ({ + entity_id: networkIdsBySlug[network.slug], + tag_id: tagIdsBySlug[tagSlug], + inherit: true, + })) + : [])) + .flat(); - await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex); - }); + await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex); + + const entities = await knex('entities').select('id', 'slug', 'type'); + + await redis.connect(); + + await redis.del('traxxx:entities:id_by_slug'); + await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id])); + + await redis.disconnect(); +}; diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 96cedf17..d37e1606 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -1,4 +1,12 @@ const upsert = require('../src/utils/upsert'); +const redis = require('../src/redis'); + +const entityPrefixes = { + channel: '', + network: '_', + studio: '*', + info: '@', +}; /* eslint-disable max-len */ const sites = [ @@ -10554,6 +10562,18 @@ const sites = [ siteAsSerie: true, }, }, + { + name: 'Hardwerk', + slug: 'hardwerk', + url: 'https://hardwerk.com', + independent: true, + parent: 'radical', + parameters: { + endpoint: 'jC4SrjH8YVDtRejiA0PMx', + videos: 'films', + actors: 'performers', + }, + }, // REALITY KINGS { name: 'Look At Her Now', @@ -15540,195 +15560,86 @@ sites.reduce((acc, site) => { }, new Set()); /* eslint-disable max-len */ -exports.seed = (knex) => Promise.resolve() - .then(async () => { - await Promise.all(sites.map(async (channel) => { - if (channel.rename) { - await knex('entities') - .where({ - type: channel.type || 'channel', - slug: channel.rename, - }) - .update('slug', channel.slug); +exports.seed = async (knex) => { + await Promise.all(sites.map(async (channel) => { + if (channel.rename) { + await knex('entities') + .where({ + type: channel.type || 'channel', + slug: channel.rename, + }) + .update('slug', channel.slug); - return; + return; + } + + if (channel.delete) { + await knex('entities') + .where({ + type: channel.type || 'channel', + slug: channel.slug, + }) + .delete(); + } + }).filter(Boolean)); + + const networks = await knex('entities') + .where('type', 'network') + .orWhereNull('parent_id'); + + const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); + + const tags = await knex('tags').select('*').whereNull('alias_for'); + const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); + + const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({ + slug: site.slug, + name: site.name, + name_stylized: site.style, + type: site.type || 'channel', + alias: site.alias, + description: site.description, + url: site.url, + parameters: site.parameters || null, + options: site.options, + parent_id: networksMap[site.parent] || null, + priority: site.priority || 0, + independent: !!site.independent, + visible: site.visible, + showcased: site.showcased, + has_logo: site.hasLogo === undefined ? true : site.hasLogo, + })); + + const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex); + const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); + + const tagAssociations = sites.map((site) => (site.tags && !site.delete + ? site.tags.map((tagSlug) => { + const tag = tagsMap[tagSlug]; + + if (!tag) { + console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`); } - if (channel.delete) { - await knex('entities') - .where({ - type: channel.type || 'channel', - slug: channel.slug, - }) - .delete(); - } - }).filter(Boolean)); + return { + entity_id: sitesMap[site.slug], + tag_id: tagsMap[tagSlug], + inherit: true, + }; + }) + : [] + )).flat(); - const networks = await knex('entities') - .where('type', 'network') - .orWhereNull('parent_id'); + await upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex); - const networksMap = networks.filter((network) => !network.delete).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); + const entities = await knex('entities').select('id', 'slug', 'type'); - const tags = await knex('tags').select('*').whereNull('alias_for'); - const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); + await redis.connect(); - const sitesWithNetworks = sites.filter((site) => !site.delete).map((site) => ({ - slug: site.slug, - name: site.name, - name_stylized: site.style, - type: site.type || 'channel', - alias: site.alias, - description: site.description, - url: site.url, - parameters: site.parameters || null, - options: site.options, - parent_id: networksMap[site.parent] || null, - priority: site.priority || 0, - independent: !!site.independent, - visible: site.visible, - showcased: site.showcased, - has_logo: site.hasLogo === undefined ? true : site.hasLogo, - })); + await redis.del('traxxx:entities:id_by_slug'); + await redis.hSet('traxxx:entities:id_by_slug', entities.map((entity) => [`${entityPrefixes[entity.type]}${entity.slug}`, entity.id])); - const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex); - const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); - - const tagAssociations = sites.map((site) => (site.tags && !site.delete - ? site.tags.map((tagSlug) => { - const tag = tagsMap[tagSlug]; - - if (!tag) { - console.warn(`Tag ${tagSlug} for ${site.slug} does not exist`); - } - - return { - entity_id: sitesMap[site.slug], - tag_id: tagsMap[tagSlug], - inherit: true, - }; - }) - : [] - )).flat(); - - return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex); - }); + await redis.disconnect(); +}; exports.sites = sites; - -/* - 'X-Art' => 'xart', - 'met-art' => 'metart', - '18og' => '18OnlyGirls', - 'a1o1' => 'Asian1on1', - 'add' => 'ManualAddActors', - 'analb' => 'AnalBeauty', - 'bgonzo' => 'BangGonzo', - 'btlbd' => 'BigTitsLikeBigDicks', - 'bjf' => 'BlowjobFridays', - 'cws' => 'CzechWifeSwap', - 'Daughter' => 'DaughterSwap', - 'Daughters' => 'DaughterSwap', - 'dc' => 'DorcelVision', - 'dpg' => 'DigitalPlayground', - 'dsw' => 'DaughterSwap', - 'faq' => 'FirstAnalQuest', - 'ft' => 'FastTimes', - 'fittingroom' => 'Fitting-Room', - 'gbcp' => 'GangbangCreampie', - 'hart' => 'Hegre', - 'hegre-art' => 'Hegre', - 'kha' => 'KarupsHA', - 'kow' => 'KarupsOW', - 'kpc' => 'KarupsPC', - 'la' => 'LatinAdultery', - 'lcd' => 'LittleCaprice', - 'lhf' => 'LoveHerFeet', - 'littlecapricedreams' => 'Little Caprice Dreams', - 'maj' => 'ManoJob', - 'mfl' => 'Mofos', - 'mj' => 'ManoJob', - 'mpov' => 'MrPOV', - 'naughtyamericavr' => 'NaughtyAmerica', - 'news' => 'NewSensations', - 'ps' => 'PropertySex', - 'sart' => 'SexArt', - 'sbj' => 'StreetBlowjobs', - 'sislove' => 'SisLovesMe', - 'tds' => 'TheDickSuckers', - 'these' => 'TheStripperExperience', - 'tlc' => 'TeensLoveCream', - 'tle' => 'TheLifeErotic', - 'tog' => 'TonightsGirlfriend', - 'wowg' => 'WowGirls', - 'wy' => 'WebYoung', - 'itc' => 'InTheCrack', - "abbw" => "AbbyWinters", - "abme" => "AbuseMe", - "ana" => "AnalAngels", - "atke" => "ATKExotics", - "atkg" => "ATKGalleria", - "atkgfs" => "ATKGirlfriends", - "atkh" => "ATKHairy", - "aktp" => "ATKPetites", - "ba" => "Beauty-Angels", - "bna" => "BrandNew", - "bam" => "BruceAndMorgan", - "bcast" => "BrutalCastings", - "bd" => "BrutalDildos", - "bpu" => "BrutalPickups", - "cza" => "CzhecAmateurs", - "czbb" => "CzechBangBus", - "czb" => "CzechBitch", - "cc" => "CzechCasting", - "czc" => "CzechCouples", - "czestro" => "CzechEstrogenolit", - "czf" => "CzechFantasy", - "czgb" => "CzechGangBang", - "cgfs" => "CzechGFS", - "czharem" => "CzechHarem", - "czm" => "CzechMassage", - "czo" => "CzechOrgasm", - "czps" => "CzechPawnShop", - "css" => "CzechStreets", - "cztaxi" => "CzechTaxi", - "czt" => "CzechTwins", - "dts" => "DeepThroatSirens", - "doan" => "DiaryOfANanny", - "ds" => "DungeonSex", - "ffr" => "FacialsForever", - "ff" => "FilthyFamily", - "fbbg" => "FirstBGG", - "fs" => "FuckStudies", - "tfcp" => "FullyClothedPissing", - "gdp" => "GirlsDoPorn", - "Harmony" => "HarmonyVision", - "hletee" => "HelplessTeens", - "jlmf" => "JessieLoadsMonsterFacials", - "lang" => "LANewGirl", - "mmp" => "MMPNetwork", - "mbc" => "MyBabysittersClub", - "nvg" => "NetVideoGirls", - "oo" => "Only-Opaques", - "os" => "Only-Secretaries", - "oss" => "OnlySilAndSatin", - "psus" => "PascalsSubSluts", - "psp" => "PorsntarsPunishment", - "pdmqfo" => "QuestForOrgasm", - "sed" => "SexualDisgrace", - "sislov" => "SisLovesMe", - "tslw" => "SlimeWave", - "stre" => "StrictRestraint", - "t18" => "Taboo18", - "tsma" => "TeenSexMania", - "tsm" => "TeenSexMovs", - "ttw" => "TeensInTheWoods", - "tgw" => "ThaiGirlsWild", - "taob" => "TheArtOfBlowJob", - "trwo" => "TheRealWorkout", - "tt" => "TryTeens", - "vp" => "VIPissy", - "wrh" => "WeAreHairy", - "yt" => "YoungThroats", - ]; -*/ diff --git a/seeds/06_affiliates.js b/seeds/06_affiliates.js index 65116896..fad4cfa1 100755 --- a/seeds/06_affiliates.js +++ b/seeds/06_affiliates.js @@ -719,6 +719,11 @@ const affiliates = [ url: 'https://register.join-toughlovex.com/track/MzAwMDA5NzkuMy43Ni4xOTcuMC4wLjAuMC4w', comment: 'rev share', }, + { + channel: 'hardwerk', + url: 'https://register.hardwerk.com/track/MzAwMDA5NzkuMy4xNTEuMzM5LjAuMC4wLjAuMA', + comment: 'rev share', + }, // radical > topwebmodels { network: 'topwebmodels', diff --git a/src/scrapers/actors.js b/src/scrapers/actors.js index 329a193e..6fca5fd2 100644 --- a/src/scrapers/actors.js +++ b/src/scrapers/actors.js @@ -140,6 +140,7 @@ module.exports = { purgatoryx: radical, topwebmodels: radical, lucidflix: radical, + hardwerk: radical, // hush / hussiepass eyeontheguy: hush, hushpass: hush, diff --git a/src/scrapers/radical.js b/src/scrapers/radical.js index be2ba179..03ae08bd 100755 --- a/src/scrapers/radical.js +++ b/src/scrapers/radical.js @@ -215,7 +215,7 @@ function scrapeProfile(data, channel, scenes, parameters) { async function fetchProfile(actor, { channel, parameters }) { const endpoint = await fetchEndpoint(channel); - const res = await http.get(`${channel.url}/_next/data/${endpoint}/models/${actor.slug}.json?slug=${actor.slug}`); + const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`); if (res.ok && res.body.pageProps?.model) { return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters); diff --git a/tests/profiles.js b/tests/profiles.js index b111b88b..34edaa57 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -153,6 +153,7 @@ const actors = [ { entity: 'topwebmodels', name: 'Lexi Belle', fields: ['avatar', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'purgatoryx', name: 'Kenzie Reeves', fields: ['avatar', 'description', 'gender', 'dateOfBirth', 'birthPlace', 'measurements', 'height', 'weight', 'eyes', 'hairColor'] }, { entity: 'lucidflix', name: 'Ava Amira', fields: ['avatar', 'description', 'gender'] }, + { entity: 'hardwerk', name: 'Luna Silver', fields: ['avatar', 'gender'] }, // wankz { entity: 'wankzvr', name: 'Melody Marks', fields: ['avatar', 'gender', 'description', 'birthPlace', 'height', 'measurements', 'age'] }, { entity: 'milfvr', name: 'Ember Snow', fields: ['avatar', 'gender', 'description', 'measurements', 'birthPlace', 'height', 'age'] },