DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 35 Pull Requests 1 Releases Wiki Activity

Excluding studios from entity aggregration. Added profile scraping to Teste de Fudelidade.

This commit is contained in:
DebaucheryLibrarian 2023-08-04 22:30:18 +02:00
parent 328550b9a3
commit 5f26f8de27
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/entities.js
View File

@ -206,8 +206,9 @@ async function fetchEntitiesBySlug(entitySlugs, prefer = 'channel') {
parent_id,
array['parent'] as parent_path
FROM entities
WHERE slug = ANY(:entitySlugs)
OR url ILIKE ANY(:entityHosts)
WHERE (slug = ANY(:entitySlugs)
OR url ILIKE ANY(:entityHosts))
AND type IN ('channel', 'network')
UNION ALL

8
src/scrapers/testedefudelidade.js
View File

@ -71,7 +71,6 @@ function scrapeProfile({ query }, url) {
profile.avatar = query.img('.model .photo img');
profile.scenes = scrapeAll(unprint.initAll(query.all('.single .itemsingle')));
console.log(profile);
return profile;
}
@ -89,8 +88,11 @@ async function fetchLatest(channel, page) {
}
async function fetchProfile(actor, { entity }) {
const url = actor.url || `${entity.url}/models/${actor.slug}/`;
const res = await unprint.get(url);
const url = actor.url || `${entity.url}/modelos/${actor.slug}/`;
const res = await unprint.get(url, {
maxRedirects: 0, // will attempt to redirect invalid URL to relevant scene page
});
if (res.ok) {
return scrapeProfile(res.context, url);