DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Updated Kink profile scraper.

This commit is contained in:
DebaucheryLibrarian 2023-01-02 00:54:10 +01:00
parent eb759a0dec
commit 45715773f1
1 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/scrapers/kink.js
View File

@ -91,29 +91,38 @@ function scrapeScene({ query }, url, entity) {
return release;
}
async function fetchActorReleases(actorUrl, page = 1, accReleases = []) {
const res = await qu.get(`${actorUrl}?page=${page}`);
async function fetchActorReleases(actorId, entity, page = 1, accReleases = []) {
const networkUrl = entity.type === 'channel' ? entity.parent.url : entity.url;
const { tab } = await http.getBrowserSession('kink');
const res = await tab.goto(`${networkUrl}/search?type=shoots&performerIds=${actorId}&sort=published&page=${page}`);
if (res.ok) {
const releases = scrapeAll(qu.initAll(res.item.el, '.shoot-list .shoot'));
const hasNextPage = res.item.query.exists('.paginated-nav li:last-child:not(.disabled)');
if (res.status() === 200) {
const html = await tab.content();
const item = unprint.init(html);
const releases = scrapeAll(unprint.initAll(html, '.results .shoot-card'), entity);
const hasNextPage = item.query.exists('.paginated-nav li:last-child:not(.disabled)');
await tab.close();
if (hasNextPage) {
return fetchActorReleases(actorUrl, page + 1, accReleases.concat(releases));
return fetchActorReleases(actorId, entity, page + 1, accReleases.concat(releases));
}
return accReleases.concat(releases);
}
await tab.close();
return accReleases;
}
async function scrapeProfile({ query }, actorUrl, include) {
async function scrapeProfile({ query }, actorUrl, entity, include) {
const profile = {};
profile.description = query.content('.bio #expand-text');
profile.entryId = actorUrl.match(/\/model\/(\d+)\//)?.[1] || query.attribute('.favorite-button.bio-favorite', 'data-id');
profile.description = query.content('.bio-outer #expand-text');
const tags = query.contents('.bio-tags a');
const tags = query.contents('.bio-tags a').map((tag) => tag.toLowerCase());
if (tags.includes('brunette') || tags.includes('brunet')) profile.hairColor = 'brown';
if (tags.includes('blonde') || tags.includes('blond')) profile.hairColor = 'blonde';
@ -138,12 +147,10 @@ async function scrapeProfile({ query }, actorUrl, include) {
profile.avatar = query.img('.bio-slider-img, .bio-img:not([src*="Missing"])');
profile.social = query.urls('a.social-link');
if (include.releases) {
profile.releases = await fetchActorReleases(actorUrl);
if (include.releases && profile.entryId) {
profile.releases = await fetchActorReleases(profile.entryId, entity);
}
console.log(profile);
return profile;
}
@ -215,7 +222,7 @@ async function fetchProfile({ name: actorName }, entity, options) {
await tab.close();
return scrapeProfile(item, actorUrl, options);
return scrapeProfile(item, actorUrl, entity, options);
}
await tab.close();