DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Storing actor socials, improved Hush profile scraper.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2023-07-23 01:02:18 +02:00
parent 48acabac49
commit ca695db3ba
5 changed files with 68 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/actors.js
View File

@@ -775,7 +775,41 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
}
async function associateSocials(profiles) {
console.log('profiles', profiles);
const profileEntries = await knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profiles.map((profile) => [profile.id, profile.entity.id]));
const profileEntriesByActorIdAndEntityId = profileEntries.reduce((acc, profileEntry) => {
if (!acc[profileEntry.actor_id]) {
acc[profileEntry.actor_id] = {};
}
acc[profileEntry.actor_id][profileEntry.entity_id] = profileEntry.id;
return acc;
}, {});
profiles.reduce(async (chain, profile) => {
await chain;
if (!Array.isArray(profile.social) || profile.social.length === 0) {
return;
}
const profileId = profileEntriesByActorIdAndEntityId[profile.id]?.[profile.entity.id];
if (!profileId) {
return;
}
await knex('actors_social')
.insert(profile.social.map((url) => ({
url,
platform: new URL(url).hostname.match(/([\w-]+)?\.(\w+)$/)?.[1],
actor_id: profile.id,
profile_id: profileId,
})))
.onConflict()
.ignore();
}, Promise.resolve());
}
async function getActorNames(actorNames) {
@@ -801,7 +835,7 @@ async function storeProfiles(profiles) {
const profilesWithAvatarIds = await associateAvatars(profiles);
const actorIds = Array.from(new Set(profiles.map((profile) => profile.id)));
// await associateSocials(profiles);
await associateSocials(profiles);
await upsertProfiles(profilesWithAvatarIds);
await interpolateProfiles(actorIds);