diff --git a/assets/components/actors/actors.vue b/assets/components/actors/actors.vue index a99f033b..25d54f6f 100644 --- a/assets/components/actors/actors.vue +++ b/assets/components/actors/actors.vue @@ -272,7 +272,7 @@ import Pagination from '../pagination/pagination.vue'; const toggleValues = [true, null, false]; const boobSizes = 'ABCDEFGHIJKZ'.split(''); -const topCountries = ['AU', 'BR', 'DE', 'JP', 'RU', 'GB', 'US']; +const topCountries = ['AU', 'BR', 'CZ', 'DE', 'JP', 'RU', 'GB', 'US']; function updateFilters() { this.$router.push({ diff --git a/public/img/logos/purgatoryx/favicon.png b/public/img/logos/purgatoryx/favicon.png new file mode 100644 index 00000000..ac5a886f Binary files /dev/null and b/public/img/logos/purgatoryx/favicon.png differ diff --git a/public/img/logos/purgatoryx/favicon_dark.png b/public/img/logos/purgatoryx/favicon_dark.png new file mode 100644 index 00000000..c1e270fc Binary files /dev/null and b/public/img/logos/purgatoryx/favicon_dark.png differ diff --git a/public/img/logos/purgatoryx/favicon_light.png b/public/img/logos/purgatoryx/favicon_light.png new file mode 100644 index 00000000..7b791263 Binary files /dev/null and b/public/img/logos/purgatoryx/favicon_light.png differ diff --git a/public/img/logos/purgatoryx/lazy/favicon.png b/public/img/logos/purgatoryx/lazy/favicon.png new file mode 100644 index 00000000..391acac9 Binary files /dev/null and b/public/img/logos/purgatoryx/lazy/favicon.png differ diff --git a/public/img/logos/purgatoryx/lazy/favicon_dark.png b/public/img/logos/purgatoryx/lazy/favicon_dark.png new file mode 100644 index 00000000..85e9da57 Binary files /dev/null and b/public/img/logos/purgatoryx/lazy/favicon_dark.png differ diff --git a/public/img/logos/purgatoryx/lazy/favicon_light.png b/public/img/logos/purgatoryx/lazy/favicon_light.png new file mode 100644 index 00000000..9a33f267 Binary files /dev/null and b/public/img/logos/purgatoryx/lazy/favicon_light.png differ diff --git a/public/img/logos/purgatoryx/lazy/network.png b/public/img/logos/purgatoryx/lazy/network.png new file mode 100644 index 00000000..c26045b7 Binary files /dev/null and b/public/img/logos/purgatoryx/lazy/network.png differ diff --git a/public/img/logos/purgatoryx/lazy/purgatoryx.png b/public/img/logos/purgatoryx/lazy/purgatoryx.png new file mode 100644 index 00000000..8986b63e Binary files /dev/null and b/public/img/logos/purgatoryx/lazy/purgatoryx.png differ diff --git a/public/img/logos/purgatoryx/network.png b/public/img/logos/purgatoryx/network.png new file mode 100644 index 00000000..082c51bf Binary files /dev/null and b/public/img/logos/purgatoryx/network.png differ diff --git a/public/img/logos/purgatoryx/purgatoryx.png b/public/img/logos/purgatoryx/purgatoryx.png new file mode 100644 index 00000000..57c615de Binary files /dev/null and b/public/img/logos/purgatoryx/purgatoryx.png differ diff --git a/public/img/logos/purgatoryx/thumbs/favicon.png b/public/img/logos/purgatoryx/thumbs/favicon.png new file mode 100644 index 00000000..391acac9 Binary files /dev/null and b/public/img/logos/purgatoryx/thumbs/favicon.png differ diff --git a/public/img/logos/purgatoryx/thumbs/favicon_dark.png b/public/img/logos/purgatoryx/thumbs/favicon_dark.png new file mode 100644 index 00000000..85e9da57 Binary files /dev/null and b/public/img/logos/purgatoryx/thumbs/favicon_dark.png differ diff --git a/public/img/logos/purgatoryx/thumbs/favicon_light.png b/public/img/logos/purgatoryx/thumbs/favicon_light.png new file mode 100644 index 00000000..9a33f267 Binary files /dev/null and b/public/img/logos/purgatoryx/thumbs/favicon_light.png differ diff --git a/public/img/logos/purgatoryx/thumbs/network.png b/public/img/logos/purgatoryx/thumbs/network.png new file mode 100644 index 00000000..2ed82091 Binary files /dev/null and b/public/img/logos/purgatoryx/thumbs/network.png differ diff --git a/public/img/logos/purgatoryx/thumbs/purgatoryx.png b/public/img/logos/purgatoryx/thumbs/purgatoryx.png new file mode 100644 index 00000000..ee47f81b Binary files /dev/null and b/public/img/logos/purgatoryx/thumbs/purgatoryx.png differ diff --git a/public/img/logos/radical/lazy/network.png b/public/img/logos/radical/lazy/network.png new file mode 100644 index 00000000..54383de5 Binary files /dev/null and b/public/img/logos/radical/lazy/network.png differ diff --git a/public/img/logos/radical/lazy/radical.png b/public/img/logos/radical/lazy/radical.png new file mode 100644 index 00000000..6718666e Binary files /dev/null and b/public/img/logos/radical/lazy/radical.png differ diff --git a/public/img/logos/radical/misc/radical-cash.png b/public/img/logos/radical/misc/radical-cash.png new file mode 100644 index 00000000..b1f9798a Binary files /dev/null and b/public/img/logos/radical/misc/radical-cash.png differ diff --git a/public/img/logos/radical/misc/radical-dark.png b/public/img/logos/radical/misc/radical-dark.png new file mode 100644 index 00000000..a48947b2 Binary files /dev/null and b/public/img/logos/radical/misc/radical-dark.png differ diff --git a/public/img/logos/radical/misc/radical-light.png b/public/img/logos/radical/misc/radical-light.png new file mode 100644 index 00000000..0ea53154 Binary files /dev/null and b/public/img/logos/radical/misc/radical-light.png differ diff --git a/public/img/logos/radical/network.png b/public/img/logos/radical/network.png new file mode 100644 index 00000000..28b59fce Binary files /dev/null and b/public/img/logos/radical/network.png differ diff --git a/public/img/logos/radical/radical.png b/public/img/logos/radical/radical.png new file mode 100644 index 00000000..7bf1e92c Binary files /dev/null and b/public/img/logos/radical/radical.png differ diff --git a/public/img/logos/radical/thumbs/network.png b/public/img/logos/radical/thumbs/network.png new file mode 100644 index 00000000..d6a90acd Binary files /dev/null and b/public/img/logos/radical/thumbs/network.png differ diff --git a/public/img/logos/radical/thumbs/radical.png b/public/img/logos/radical/thumbs/radical.png new file mode 100644 index 00000000..a9ec48c7 Binary files /dev/null and b/public/img/logos/radical/thumbs/radical.png differ diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 450b3631..8d22b0cc 100644 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -52,6 +52,12 @@ const parentNetworks = [ }, parent: 'gamma', }, + { + slug: 'radical', + alias: ['kb productions'], + name: 'Radical Entertainment', + url: 'https://radicalcash.com', + }, ]; const networks = [ @@ -550,6 +556,7 @@ const networks = [ slug: 'topwebmodels', name: 'Top Web Models', url: 'https://tour.topwebmodels.com', + parent: 'radical', parameters: { apiKey: '5b637cd8c4bc59cd13686f1c38dcb780', }, @@ -624,12 +631,12 @@ const networks = [ }, ]; -exports.seed = knex => Promise.resolve() +exports.seed = (knex) => Promise.resolve() .then(async () => { - const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map(network => ({ ...network, type: 'network' })), ['slug', 'type'], knex); + const grandParentNetworkEntries = await upsert('entities', grandParentNetworks.map((network) => ({ ...network, type: 'network' })), ['slug', 'type'], knex); const grandParentNetworksBySlug = [].concat(grandParentNetworkEntries.inserted, grandParentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const parentNetworksWithGrandParent = parentNetworks.map(network => ({ + const parentNetworksWithGrandParent = parentNetworks.map((network) => ({ slug: network.slug, name: network.name, type: network.type || 'network', @@ -643,7 +650,7 @@ exports.seed = knex => Promise.resolve() const parentNetworkEntries = await upsert('entities', parentNetworksWithGrandParent, ['slug', 'type'], knex); const parentNetworksBySlug = [].concat(parentNetworkEntries.inserted, parentNetworkEntries.updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const networksWithParent = networks.map(network => ({ + const networksWithParent = networks.map((network) => ({ slug: network.slug, name: network.name, type: network.type || 'network', @@ -665,14 +672,14 @@ exports.seed = knex => Promise.resolve() networkEntries.updated, ).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); - const tagSlugs = networks.map(network => network.tags).flat().filter(Boolean); + const tagSlugs = networks.map((network) => network.tags).flat().filter(Boolean); const tagEntries = await knex('tags').whereIn('slug', tagSlugs); const tagIdsBySlug = tagEntries.reduce((acc, tag) => ({ ...acc, [tag.slug]: tag.id }), {}); const tagAssociations = networks - .map(network => (network.tags - ? network.tags.map(tagSlug => ({ + .map((network) => (network.tags + ? network.tags.map((tagSlug) => ({ entity_id: networkIdsBySlug[network.slug], tag_id: tagIdsBySlug[tagSlug], inherit: true, diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 128cc17e..a9f499c6 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -7235,6 +7235,14 @@ const sites = [ parent: 'puretaboo', }, */ + // RADICAL ENTERTAINMENT + { + name: 'PurgatoryX', + slug: 'purgatoryx', + url: 'https://tour.purgatoryx.com', + independent: true, + parent: 'radical', + }, // REALITY KINGS { name: 'Look At Her Now', @@ -10706,7 +10714,7 @@ const sites = [ ]; /* eslint-disable max-len */ -exports.seed = knex => Promise.resolve() +exports.seed = (knex) => Promise.resolve() .then(async () => { const networks = await knex('entities') .where('type', 'network') @@ -10717,7 +10725,7 @@ exports.seed = knex => Promise.resolve() const tags = await knex('tags').select('*').whereNull('alias_for'); const tagsMap = tags.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); - const sitesWithNetworks = sites.map(site => ({ + const sitesWithNetworks = sites.map((site) => ({ slug: site.slug, name: site.name, type: site.type || 'channel', @@ -10735,8 +10743,8 @@ exports.seed = knex => Promise.resolve() const { inserted, updated } = await upsert('entities', sitesWithNetworks, ['slug', 'type'], knex); const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); - const tagAssociations = sites.map(site => (site.tags - ? site.tags.map(tagSlug => ({ + const tagAssociations = sites.map((site) => (site.tags + ? site.tags.map((tagSlug) => ({ entity_id: sitesMap[site.slug], tag_id: tagsMap[tagSlug], inherit: true, diff --git a/src/deep.js b/src/deep.js index 5af3b1c7..af3e4383 100644 --- a/src/deep.js +++ b/src/deep.js @@ -178,7 +178,7 @@ async function scrapeReleases(baseReleases, entitiesBySlug, type) { return [slug, { ...entity, preData }]; } - return null; + return [slug, entity]; })); const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean)); diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index 9c4917b8..5d7e38a6 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -179,6 +179,8 @@ async function getSession(site, parameters) { const cookieString = await cookieJar.getCookieStringAsync(sessionUrl); const { instance_token: instanceToken } = cookie.parse(cookieString); + console.log(site.name, instanceToken); + return { session, instanceToken }; } @@ -240,6 +242,8 @@ async function fetchLatest(site, page = 1, options) { const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); + console.log('fetch', instanceToken); + const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const limit = 24; const apiUrl = site.parameters?.native || site.parameters?.extract diff --git a/src/scrapers/purgatoryx.js b/src/scrapers/purgatoryx.js new file mode 100644 index 00000000..a824a374 --- /dev/null +++ b/src/scrapers/purgatoryx.js @@ -0,0 +1,152 @@ +'use strict'; + +const qu = require('../utils/qu'); +const http = require('../utils/http'); +const slugify = require('../utils/slugify'); +const { feetInchesToCm, lbsToKg } = require('../utils/convert'); + +function scrapeAll(scenes) { + return scenes.map(({ query }) => { + const release = {}; + + release.title = query.cnt('.title'); + release.url = query.url('.title a'); + release.entryId = new URL(release.url).pathname.match(/\/view\/(\d+)/)[1]; + + release.date = query.date('.pub-date', 'MMM DD, YYYY'); + release.duration = query.duration('.video-duration'); + + release.actors = query.all('.models a').map((el) => ({ + name: query.cnt(el), + url: query.url(el, null), + })); + + if (query.exists('.thumb-big')) { // updates page + release.poster = query.img('.thumb-big', 'data-image') || JSON.parse(query.el('.thumbnails-wrap a', 'data-images')); + release.photos = [query.img('.thumb-top', 'data-image'), query.img('.thumb-bottom', 'data-image')]; + } + + if (query.exists('.thumbnails-wrap')) { // actor page + try { + const images = JSON.parse(query.el('.thumbnails-wrap a', 'data-images')); + + release.poster = images.slice(0, 1)[0]; + release.photos = images.slice(1); + } catch (error) { + // images probably not available + } + } + + console.log(release.photos); + + return release; + }); +} + +function scrapeScene({ query }, url) { + const release = {}; + + release.title = query.cnt('.title'); + release.entryId = new URL(url).pathname.match(/\/view\/(\d+)/)[1]; + release.date = query.date('.date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/); + + release.description = query.cnt('.description p'); + release.duration = query.duration('.total-time'); + + release.actors = query.all('.model-wrap li').map((el) => ({ + name: query.cnt(el, 'h5'), + url: query.url(el, 'a'), + avatar: query.img(el), + })); + + release.poster = query.poster(); + release.photos = query.urls('.photos-slider a'); + release.trailer = query.video(); + + release.comment = query.cnt('.series'); + + console.log(release); + + return release; +} + +async function fetchLatest(channel, page) { + const res = await qu.getAll(`${channel.url}/episodes?page=${page}`, '.content-item'); + + if (res.ok) { + return scrapeAll(res.items, channel); + } + + return res.status; +} + +function scrapeProfile({ query }, url) { + const profile = { url }; + + const bio = Object.fromEntries(query.all('.model-desc li').map((el) => [slugify(query.cnt(el, 'span'), '_'), query.text(el)])); + + profile.description = bio.bio; + + profile.dateOfBirth = qu.extractDate(bio.birthdate, 'YYYY-MM-DD'); + profile.birthPlace = bio.birthplace; + + profile.hairColor = bio.hair_color; + profile.eyes = bio.eye_color; + + profile.height = feetInchesToCm(bio.height); + profile.weight = lbsToKg(bio.weight); + profile.measurements = bio.measurements; + + profile.avatar = query.img('.model-pic img'); + + profile.scenes = scrapeAll(qu.initAll(query.all('.content-item'))); + + console.log(bio); + console.log(profile); + + return profile; +} + +async function searchActor(baseActor, channel) { + const searchRes = await http.post(`${channel.url}/search-preview`, { q: slugify(baseActor.name, ' ') }, { + encodeJSON: false, + headers: { + 'Accept-Language': 'en-US,en;', + }, + }); + + if (searchRes.ok) { + const actorUrl = searchRes.body.find((item) => item.type === 'model' && slugify(item.title) === baseActor.slug)?.url; + + return actorUrl || null; + } + + return null; +} + +async function fetchProfile(baseActor, context, include, retry = false) { + const actorUrl = (!retry && baseActor.url) || await searchActor(baseActor, context.entity); + + if (!actorUrl) { + return null; + } + + const res = await qu.get(actorUrl); + + if (res.ok) { + return scrapeProfile(res.item, actorUrl); + } + + if (baseActor.url) { + return fetchProfile(baseActor, context, include, true); + } + + return res.status; +} + +module.exports = { + fetchLatest, + fetchProfile, + scrapeAll, + scrapeScene, +}; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index f80afe28..677a48c2 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -51,6 +51,7 @@ const pascalssubsluts = require('./pascalssubsluts'); // reserved keyword const pierrewoodman = require('./pierrewoodman'); const pinkyxxx = require('./pinkyxxx'); const privateNetwork = require('./private'); // reserved keyword +const purgatoryx = require('./purgatoryx'); // reserved keyword const score = require('./score'); const spizoo = require('./spizoo'); const teamskeet = require('./teamskeet'); @@ -136,6 +137,7 @@ const scrapers = { porncz, pornpros: whalemember, private: privateNetwork, + purgatoryx, score, sexyhub: mindgeek, spizoo, @@ -255,6 +257,7 @@ const scrapers = { povperverts: fullpornnetwork, povpornstars: hush, private: privateNetwork, + purgatoryx, realitykings: mindgeek, realvr: badoink, roccosiffredi: famedigital, diff --git a/src/updates.js b/src/updates.js index 5cff3b9b..e783d53a 100644 --- a/src/updates.js +++ b/src/updates.js @@ -39,9 +39,11 @@ function filterLocalUniqueReleases(releases, accReleases) { async function filterUniqueReleases(releases) { const releaseIdentifiers = releases - .map((release) => [release.entity.id, release.entryId]); + .map((release) => [release.entity.id, release.entryId.toString()]); - const duplicateReleaseEntries = await knex('releases') + console.log(releaseIdentifiers.length); + + const duplicateReleaseEntriesQuery = knex('releases') .select(knex.raw('releases.*, row_to_json(entities) as entity')) .leftJoin('entities', 'entities.id', 'releases.entity_id') .whereIn(['entity_id', 'entry_id'], releaseIdentifiers) @@ -55,6 +57,10 @@ async function filterUniqueReleases(releases) { .orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updates expected }); + console.log(duplicateReleaseEntriesQuery.toString()); + + const duplicateReleaseEntries = await duplicateReleaseEntriesQuery; + const duplicateReleases = duplicateReleaseEntries.map((release) => curateRelease(release)); const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {}); @@ -262,7 +268,7 @@ async function scrapeNetworkSequential(networkEntity) { } async function scrapeNetworkParallel(networkEntity) { - const beforeNetwork = await networkEntity.scraper.beforeNetwork?.(networkEntity); + const beforeNetwork = await networkEntity.scraper?.beforeNetwork?.(networkEntity); return Promise.map( networkEntity.includedChildren,