From 5f26f8de27d52201f5cec3471483f498f46b77f7 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 4 Aug 2023 22:30:18 +0200 Subject: [PATCH] Excluding studios from entity aggregration. Added profile scraping to Teste de Fudelidade. --- src/entities.js | 5 +++-- src/scrapers/testedefudelidade.js | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/entities.js b/src/entities.js index 7a3a10a27..c8af58766 100755 --- a/src/entities.js +++ b/src/entities.js @@ -206,8 +206,9 @@ async function fetchEntitiesBySlug(entitySlugs, prefer = 'channel') { parent_id, array['parent'] as parent_path FROM entities - WHERE slug = ANY(:entitySlugs) - OR url ILIKE ANY(:entityHosts) + WHERE (slug = ANY(:entitySlugs) + OR url ILIKE ANY(:entityHosts)) + AND type IN ('channel', 'network') UNION ALL diff --git a/src/scrapers/testedefudelidade.js b/src/scrapers/testedefudelidade.js index b156a0811..5f1261803 100644 --- a/src/scrapers/testedefudelidade.js +++ b/src/scrapers/testedefudelidade.js @@ -71,7 +71,6 @@ function scrapeProfile({ query }, url) { profile.avatar = query.img('.model .photo img'); profile.scenes = scrapeAll(unprint.initAll(query.all('.single .itemsingle'))); - console.log(profile); return profile; } @@ -89,8 +88,11 @@ async function fetchLatest(channel, page) { } async function fetchProfile(actor, { entity }) { - const url = actor.url || `${entity.url}/models/${actor.slug}/`; - const res = await unprint.get(url); + const url = actor.url || `${entity.url}/modelos/${actor.slug}/`; + + const res = await unprint.get(url, { + maxRedirects: 0, // will attempt to redirect invalid URL to relevant scene page + }); if (res.ok) { return scrapeProfile(res.context, url);