From 80750b44dc49f07ab9ffb2be2ab5b423b6aa67ac Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Sat, 22 Jul 2023 00:49:56 +0200 Subject: [PATCH] Added Milfy to profile list, fixed Vixen not returning gracefully when profile is not found. --- src/actors.js | 31 ++++++++++++++++++++----------- src/scrapers/scrapers.js | 1 + src/scrapers/vixen.js | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/actors.js b/src/actors.js index 67715abe..51403ad2 100755 --- a/src/actors.js +++ b/src/actors.js @@ -461,17 +461,20 @@ async function curateProfile(profile, actor) { } } - curatedProfile.social = Array.isArray(profile.social) - ? profile.social.map((social) => { - try { - const { href } = new URL(social); - return href; - } catch (error) { - logger.warn(`Profile scraper for '${profile.entity.name}' returned invalid social link: ${social}`); - return null; - } - }).filter(Boolean) - : []; + curatedProfile.social = [].concat(profile.social).map((social) => { + if (!social) { + return null; + } + + try { + const { origin, pathname } = new URL(social); + + return `${origin}${pathname}`; + } catch (error) { + logger.warn(`Profile scraper for '${profile.entity.name}' returned invalid social link: ${social}`); + return null; + } + }).filter(Boolean); curatedProfile.scenes = toBaseReleases(profile.scenes || profile.releases, profile.entity, actor) // attach actor to base scene, in case it was not scraped @@ -771,6 +774,10 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy return profiles.filter(Boolean); } +async function associateSocials(profiles) { + console.log('profiles', profiles); +} + async function getActorNames(actorNames) { if (actorNames.length > 0) { return actorNames; @@ -794,6 +801,8 @@ async function storeProfiles(profiles) { const profilesWithAvatarIds = await associateAvatars(profiles); const actorIds = Array.from(new Set(profiles.map((profile) => profile.id))); + // await associateSocials(profiles); + await upsertProfiles(profilesWithAvatarIds); await interpolateProfiles(actorIds); } diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 9357bea9..d9f6b6c7 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -247,6 +247,7 @@ const scrapers = { mariskax, metrohd: mindgeek, milehighmedia: mindgeek, + milfy: vixen, milfvr: wankzvr, mofos: mindgeek, mylf: teamskeet, diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js index 76950cfd..06330473 100755 --- a/src/scrapers/vixen.js +++ b/src/scrapers/vixen.js @@ -611,7 +611,7 @@ async function fetchProfile(actor, { channel }) { }, }); - if (res.ok) { + if (res.ok && res.body.data?.model) { return scrapeProfile(res.body.data, channel); }