common

@@ -1 +1 @@
-Subproject commit 4b90a5feeccc0c6325469dcb45a8d7cceabb386a
+Subproject commit dc00c3d58af2c23530b8b3cb6704f3860fdd7d0f

config/default.js

@@ -403,23 +403,4 @@ module.exports = {
 		flushWindow: 1000,
 	},
 	titleSlugLength: 50,
-	socials: {
-		urls: {
-			cashapp: 'https://cash.app/${handle}', // eslint-disable-line no-template-curly-in-string
-			fansly: 'https://fansly.com/{handle}',
-			instagram: 'https://www.instagram.com/{handle}',
-			linktree: 'https://linktr.ee/{handle}',
-			loyalfans: 'https://www.loyalfans.com/{handle}',
-			manyvids: 'https://{handle}.manyvids.com',
-			onlyfans: 'https://onlyfans.com/{handle}',
-			pornhub: 'https://www.pornhub.com/model/{handle}',
-			reddit: 'https://www.reddit.com/u/{handle}',
-			twitter: 'https://x.com/{handle}',
-		},
-		prefix: {
-			default: '@',
-			cashapp: '$',
-			reddit: 'u/',
-		},
-	},
 };

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "traxxx",
-    "version": "1.244.89",
+    "version": "1.244.88",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
             "name": "traxxx",
-            "version": "1.244.89",
+            "version": "1.244.88",
             "license": "ISC",
             "dependencies": {
                 "@aws-sdk/client-s3": "^3.458.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "traxxx",
-    "version": "1.244.89",
+    "version": "1.244.88",
     "description": "All the latest porn releases in one place",
     "main": "src/app.js",
     "scripts": {

src/actors.js

@@ -698,61 +698,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
 	return profiles.filter(Boolean);
 }
 
-function curateSocials(socials, platformsByHostname) {
-	return socials
-		.map((social) => {
-			if (social.url) {
-				return social.url;
-			}
-
-			if (social.handle && social.platform) {
-				return social;
-			}
-
-			if (typeof social === 'string') {
-				return {
-					url: social,
-				};
-			}
-
-			return null;
-		})
-		.filter(Boolean)
-		.map((social) => {
-			if (social.handle && social.platform && /[\w-]+/.test(social.handle) && /[a-z]+/i.test(social.platform)) {
-				return {
-					platform: social.platform.toLowerCase(),
-					handle: social.handle,
-				};
-			}
-
-			if (social.url) {
-				const { hostname, pathname } = new URL(social.url);
-				const platform = platformsByHostname[hostname];
-
-				if (platform) {
-					const handle = pathname.match(new RegExp(platform.pathname.replace('{handle}', '([\\w-]+)')))?.[1];
-
-					if (handle) {
-						return {
-							platform: platform.platform,
-							handle,
-						};
-					}
-				}
-
-				return {
-					url: social.url,
-				};
-			}
-
-			throw new Error('Invalid social');
-		})
-		.filter(Boolean);
-}
-
 async function associateSocials(profiles) {
-	const { platformsByHostname } = await actorsCommon;
 	const profileEntries = await knex('actors_profiles').whereIn(['actor_id', 'entity_id'], profiles.map((profile) => [profile.actorId, profile.entity.id]));
 
 	const profileEntriesByActorIdAndEntityId = profileEntries.reduce((acc, profileEntry) => {
@@ -779,12 +725,11 @@ async function associateSocials(profiles) {
 		}
 
 		await knex('actors_socials')
-			.insert(curateSocials(profile.social, platformsByHostname).map((social) => ({
+			.insert(profile.social.map((url) => ({
-				platform: social.platform,
+				url,
-				handle: social.handle,
+				platform: new URL(url).hostname.match(/([\w-]+)?\.(\w+)$/)?.[1],
-				url: social.url,
 				actor_id: profile.actorId,
-				// profile_id: profileId,
+				profile_id: profileId,
 			})))
 			.onConflict()
 			.ignore();

src/scrapers/kink.js

@@ -2,6 +2,7 @@
 
 const unprint = require('unprint');
 
+const http = require('../utils/http');
 const slugify = require('../utils/slugify');
 const { stripQuery } = require('../utils/url');
 
@@ -39,24 +40,23 @@ function scrapeAll(scenes, entity) {
 		}));
 
 		try {
-			release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle'))
+			release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
-				.map((src) => Array.from(new Set([
+				stripQuery(src).replace('_thumb', '_full'),
-					stripQuery(src).replace('_thumb', '_full'),
+				stripQuery(src),
-					stripQuery(src),
+				src,
-					src,
+			].filter(Boolean).map((source) => ({
-				])).filter(Boolean).map((source) => ({
+				src: source,
-					src: source,
+				expectType: {
-					expectType: {
+					PNG: 'image/png',
-						PNG: 'image/png',
+				},
-					},
+			})));
-				})));
 		} catch (error) {
 			// no photos
 		}
 
 		release.trailer = `https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`;
 
-		release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
+		release.channel = slugify(query.content('.shoot-detail-legend a[href*="/channel"]'), '');
 		release.rating = query.number('.thumb-up') / 10;
 
 		return release;
@@ -64,21 +64,25 @@ function scrapeAll(scenes, entity) {
 }
 
 async function fetchLatest(channel, page = 1) {
+	const { tab } = await http.getBrowserSession('kink', { useGlobalBrowser: false, useProxy: true });
 	const url = `${channel.parent.url}/search?type=shoots&channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
+	const res = await tab.goto(url);
+	const status = res.status();
 
-	const res = await unprint.browserRequest(url, {
+	if (status === 200) {
-		selectAll: '.container .card',
+		const html = await tab.content();
-	});
+		const items = unprint.initAll(html, '.container .card');
 
-	if (res.status === 200) {
+		const scenes = scrapeAll(items, channel);
-		// const items = unprint.initAll(html, '.container .card');
 
-		const scenes = scrapeAll(res.context, channel);
+		await tab.close();
 
 		return scenes;
 	}
 
-	return res.status;
+	await tab.close();
+
+	return status;
 }
 
 function scrapeScene({ query }, url, entity) {
@@ -145,19 +149,29 @@ function scrapeScene({ query }, url, entity) {
 }
 
 async function fetchScene(url, channel) {
-	const res = await unprint.browserRequest(url);
+	const { tab } = await http.getBrowserSession('kink', { useGlobalBrowser: false, useProxy: true });
+	const res = await tab.goto(url);
 
-	if (res.status === 200) {
+	const status = res.status();
-		const scene = scrapeScene(res.context, url, channel);
+
+	if (status === 200) {
+		const html = await tab.content();
+		const item = unprint.init(html);
+
+		const scene = scrapeScene(item, url, channel);
+
+		await tab.close();
 
 		return scene;
 	}
 
-	return res.status;
+	await tab.close();
+
+	return status;
 }
 
 async function scrapeProfile({ query }, actorUrl) {
-	const profile = { url: actorUrl };
+	const profile = {};
 
 	profile.entryId = actorUrl.match(/\/model\/(\d+)\//)?.[1] || query.attribute('h1 + button[data-id]', 'data-id');
 	profile.description = query.content('.content-container #expand-text')?.trim();
@@ -190,43 +204,42 @@ async function scrapeProfile({ query }, actorUrl) {
 	return profile;
 }
 
-async function getActorUrl({ name: actorName, url }, networkUrl) {
+async function fetchProfile({ name: actorName }, entity) {
-	if (url) {
+	const networkUrl = entity.type === 'channel' ? entity.parent.url : entity.url;
-		return url;
+	const { tab } = await http.getBrowserSession('kink', { useGlobalBrowser: false, useProxy: true });
-	}
 
 	// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
-	const searchApiRes = await unprint.browserRequest(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`);
+	const searchApiRes = await tab.goto(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`);
+	const searchStatus = searchApiRes.status();
 
-	if (searchApiRes.status === 200) {
+	if (searchStatus === 200) {
-		const data = searchApiRes.context.query.json('body pre');
+		const searchHtml = await tab.content();
+		const data = unprint.init(searchHtml).query.json('body pre');
 		const actorId = data.find((actor) => actor.label === actorName)?.id;
 
 		if (actorId) {
 			const actorUrl = `${networkUrl}/model/${actorId}/${slugify(actorName)}`;
+			const actorRes = await tab.goto(actorUrl);
+			const actorStatus = actorRes.status();
 
-			return actorUrl;
+			if (actorStatus === 200) {
-		}
+				const actorHtml = await tab.content();
-	}
+				const item = unprint.init(actorHtml);
 
-	return null;
+				await tab.close();
-}
 
-async function fetchProfile(actor, entity) {
+				return scrapeProfile(item, actorUrl);
-	const networkUrl = entity.type === 'channel' ? entity.parent.url : entity.url;
+			}
-	const actorUrl = await getActorUrl(actor, networkUrl);
 
-	if (actorUrl) {
+			await tab.close();
-		const actorRes = await unprint.browserRequest(actorUrl);
 
-		if (actorRes.status === 200) {
+			return actorRes.status;
-			return scrapeProfile(actorRes.context, actorUrl);
 		}
 
-		return actorRes.status;
+		return null;
 	}
 
-	return null;
+	return searchStatus;
 }
 
 module.exports = {