DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 36 Pull Requests 1 Releases Wiki Activity

Only scraping profile source matching actor entity, changed avatar entropy cut-off from 6 to 5.5.

This commit is contained in:
DebaucheryLibrarian 2021-03-11 04:16:59 +01:00
parent 7f53f585c0
commit 00c06778ef
3 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/components/actors/photos.vue
View File

@ -45,7 +45,7 @@
<script> <script>
function photos() { function photos() {
return this.actor.photos.filter(photo => !photo.entropy || photo.entropy > 6); return this.actor.photos.filter(photo => !photo.entropy || photo.entropy > 5.5);
} }
function sfw() { function sfw() {

14
src/actors.js
View File

@ -580,7 +580,7 @@ async function interpolateProfiles(actorIdsOrNames) {
profile.avatar_media_id = actorProfiles profile.avatar_media_id = actorProfiles
.map(actorProfile => actorProfile.avatar) .map(actorProfile => actorProfile.avatar)
.filter(avatar => avatar && (avatar.entropy === null || avatar.entropy > 6)) .filter(avatar => avatar && (avatar.entropy === null || avatar.entropy > 5.5))
.sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null; .sort((avatarA, avatarB) => avatarB.height - avatarA.height)[0]?.id || null;
return profile; return profile;
@ -646,7 +646,9 @@ async function upsertProfiles(profiles) {
} }
async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) { async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesByActorEntityId) {
const profiles = Promise.map(sources, async (source) => { const validSources = actor.entity ? sources.filter(source => source === actor.entity.slug) : sources;
const profiles = Promise.map(validSources, async (source) => {
try { try {
// config may group sources to try until success // config may group sources to try until success
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => { return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
@ -768,9 +770,11 @@ async function scrapeActors(argNames) {
const [entitiesBySlug, existingActorEntries] = await Promise.all([ const [entitiesBySlug, existingActorEntries] = await Promise.all([
fetchEntitiesBySlug(entitySlugs, 'desc'), fetchEntitiesBySlug(entitySlugs, 'desc'),
knex('actors') knex('actors')
.select(['id', 'name', 'slug', 'entry_id']) .select(knex.raw('actors.id, actors.name, actors.slug, actors.entry_id, actors.entity_id, row_to_json(entities) as entity'))
.whereIn('slug', baseActors.map(baseActor => baseActor.slug)) .whereIn('actors.slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('alias_for'), .whereNull('actors.alias_for')
.leftJoin('entities', 'entities.id', 'actors.entity_id')
.groupBy('actors.id', 'entities.id'),
]); ]);
const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({ const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({

4
src/app.js
View File

@ -25,7 +25,9 @@ function logActive() {
} }
async function init() { async function init() {
logActive(); if (argv.logActive) {
logActive();
}
if (argv.server) { if (argv.server) {
await initServer(); await initServer();