diff --git a/seeds/02_sites.js b/seeds/02_sites.js index a011a0e4..6f9d9a78 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2718,7 +2718,7 @@ const sites = [ independent: true, parameters: { frontend: 1, - languages: ['nl', 'en'], + languages: ['nl', 'en', null], secret: 'HwzAace4HlS1W9h4YdPLhG6OzZjFCmIy6LUYhfv3', apiPath: '/mvh', credentials: '1-3dc13a570ff233c78a3fef0b887ad44f279683fe036ccbac8e494bb89422ed77-mvh', diff --git a/src/scrapers/actors.js b/src/scrapers/actors.js index 1aeb24ec..d2adff1e 100644 --- a/src/scrapers/actors.js +++ b/src/scrapers/actors.js @@ -206,8 +206,7 @@ module.exports = { bang, bluedonkeymedia, delphine: modelmedia, - meidenvanholland: bluedonkeymedia, - vurigvlaanderen: bluedonkeymedia, + meidenvanholland: bluedonkeymedia, // Vurig Vlaanderen uses same database boobpedia, bradmontana, cherrypimps, diff --git a/src/scrapers/bluedonkeymedia.js b/src/scrapers/bluedonkeymedia.js index 94827c3b..76b93812 100644 --- a/src/scrapers/bluedonkeymedia.js +++ b/src/scrapers/bluedonkeymedia.js @@ -42,7 +42,7 @@ async function scrapeSceneData(scene, channel, context, isDeep) { })); release.tags = scene.resources?.data?.filter((tag) => tag.type === 'category').map((tag) => tag.title); - release.language = scene.videos?.data?.film?.[0]?.language; + release.language = scene.videos?.data?.film?.[0]?.language || null; if (isDeep) { const thumb = scene.images?.data?.thumb?.[0]; @@ -80,13 +80,13 @@ function scrapeAll(scenes, channel, context) { const release = await scrapeSceneData(scene, channel, context, false); - if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) { + if (!channel.parameters.languages || channel.parameters.languages.includes(release.language)) { // all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen // the international releases should go on MVH, but the API can't filter for NL+EN, so we do it here - return { ...acc, unextracted: [...acc.unextracted, release] }; + return { ...acc, scenes: [...acc.scenes, release] }; } - return { ...acc, scenes: [...acc.scenes, release] }; + return { ...acc, unextracted: [...acc.unextracted, release] }; }, Promise.resolve({ scenes: [], unextracted: [], @@ -95,8 +95,6 @@ function scrapeAll(scenes, channel, context) { async function fetchLatest(channel, page, context) { // query seems to break if any component is left out or even moved - // const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { - // const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { headers: { Origin: channel.origin, @@ -133,34 +131,32 @@ function getLocation(model) { } async function scrapeProfile(model, { entity }) { - const actor = {}; + const profile = {}; // gender unreliable, seems to report everyone as 'vrouw' (woman) - actor.name = model.title; - actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url); + profile.entryId = model.id; + profile.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.origin); - actor.entryId = model.id; + profile.description = model.description; - actor.description = model.description; + profile.dateOfBirth = model.birth_date && model.age > 18 ? new Date(model.birth_date) : null; // sometimes seems to be profile creation date + profile.age = model.age > 18 ? model.age : null; + profile.orientation = model.sexual_orientation; - actor.dateOfBirth = model.birth_date && model.age > 18 ? new Date(model.birth_date) : null; // sometimes seems to be profile creation date - actor.age = model.age > 18 ? model.age : null; - actor.orientation = model.sexual_orientation; + profile.birthPlace = getLocation(model); - actor.birthPlace = getLocation(model); + profile.height = Number(model.length) || null; + profile.weight = Number(model.weight) || null; - actor.height = Number(model.length) || null; - actor.weight = Number(model.weight) || null; + profile.eyes = model.eye_color; + profile.hairColor = model.hair_color; - actor.eyes = model.eye_color; - actor.hairColor = model.hair_color; + profile.avatar = unprint.prefixUrl(model.images?.data?.square?.[0]?.path, 'https://cdndo.sysero.nl'); - actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl'); + profile.scenes = await Promise.all(model.video?.map(async (video) => scrapeSceneData(video, entity, false))); - actor.scenes = await Promise.all(model.video?.map(async (video) => scrapeSceneData(video, entity, false))); - - return actor; + return profile; } async function fetchProfile(actor, { entity, parameters }) { @@ -177,7 +173,7 @@ async function fetchProfile(actor, { entity, parameters }) { }); if (res.ok) { - const data = res.context.window.__NUXT__?.state?.modelStore?.model; + const data = res.context.window.__NUXT__?.state?.resourcesStore?.model; if (data) { return scrapeProfile(data, { entity }); diff --git a/tests/profiles.js b/tests/profiles.js index 5efe187d..8c21e808 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -226,6 +226,7 @@ const actors = [ { entity: 'mariskax', name: 'Honey Demon', fields: ['avatar', 'gender', 'dateOfBirth', 'placeOfBirth', 'measurements', 'height', 'weight', 'hairColor', 'eyes'] }, { entity: 'pornhub', name: 'Lexi Luna', fields: ['avatar', 'gender', 'ethnicity', 'description', 'birthPlace', 'measurements', 'naturalBoobs', 'height', 'weight', 'hairColor', 'hasPiercings', 'hasTattoos'] }, { entity: 'fullpornnetwork', name: 'Kenzie Reeves', fields: ['avatar', 'description'] }, + { entity: 'meidenvanholland', name: 'Izzy Bizzy Bang Bang', fields: ['avatar', 'description'] }, ]; const actorScrapers = scrapers.actors;