Added ManyVids API seed.

seeds/00_tags.js

@@ -762,6 +762,11 @@ const tags = [
 		slug: 'high-heels',
 		group: 'clothing',
 	},
+	{
+		name: 'hotel',
+		slug: 'hotel',
+		group: 'location',
+	},
 	{
 		name: 'humiliation',
 		slug: 'humiliation',

seeds/02_sites.js

@@ -7698,6 +7698,7 @@ const sites = [
 		url: 'https://nicoledoshi.manyvids.com',
 		hasLogo: false,
 		parent: 'manyvids',
+		delete: true,
 	},
 	// MARISKA X
 	{

seeds/09_manyvids.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const unprint = require('unprint');
+const slugify = require('../src/utils/slugify');
+
+const limit = Number(process.env.MV_LIMIT) || 1000;
+
+async function fetchCreators(page = 1, acc = []) {
+	console.log(`Fetching creators ${acc.length}/${limit}`);
+
+	// size seems to be capped at 70
+	const res = await unprint.get(`https://api.manyvids.com/search/creators/list?contentPref=1,2,3&sort=top&size=50&from=${(page - 1) * 50}`);
+
+	if (res.ok && res.data?.creators) {
+		const creators = acc.concat(res.data.creators.map((creator) => ({
+			name: creator.stageName,
+			slug: `mv.${slugify(creator.slug, '')}`,
+			url: `https://www.manyvids.com/Profile/${creator.id}/${creator.slug}/`,
+			emblem: creator.avatar?.url || null,
+		})));
+
+		if (creators.length >= limit) {
+			return creators;
+		}
+
+		return fetchCreators(page + 1, creators);
+	}
+
+	return acc;
+}
+
+exports.seed = async function (knex) {
+	const channels = await fetchCreators();
+
+	const manyvidsNetwork = await knex('entities')
+		.where('slug', 'manyvids')
+		.where('type', 'network')
+		.first();
+
+	if (!manyvidsNetwork) {
+		throw new Error('ManyVids found, did the network migration run?');
+	}
+
+	await knex('entities')
+		.insert(channels.map((channel) => ({
+			name: channel.name,
+			slug: channel.slug,
+			url: channel.url,
+			parent_id: manyvidsNetwork.id,
+			has_logo: false,
+		})))
+		.onConflict(['slug', 'type'])
+		.merge(['name', 'url']);
+
+	console.log(`Done! ${channels.length} ManyVids creators added as channels`);
+};

src/scrapers/manyvids.js

@@ -26,6 +26,7 @@ async function getTrailer(videoId) {
 	return null;
 }
 
+/*
 function getModelUrl(modelData) {
 	const slug = modelData.slug || modelData.profileUrl?.match(/\/profile\/\d+\/(\w+)/i)?.[1] || null;
 
@@ -35,6 +36,7 @@ function getModelUrl(modelData) {
 
 	return null;
 }
+*/
 
 async function scrapeScene(data, context) {
 	const release = {};
@@ -61,6 +63,7 @@ async function scrapeScene(data, context) {
 	release.date = new Date(data.launchDate);
 	release.duration = unprint.extractDuration(data.duration || data.videoDuration);
 
+	/* many channels operate more like a studio than a performer, so the channel information doesn't represent a person
 	const model = data.creator || data.model;
 
 	release.actors = [{
@@ -69,6 +72,7 @@ async function scrapeScene(data, context) {
 		avatar: model.avatar?.url || model.avatar,
 		url: getModelUrl(model),
 	}];
+	*/
 
 	release.tags = [
 		data.is3D && '3d',
@@ -86,18 +90,44 @@ async function scrapeScene(data, context) {
 
 	release.qualities = [data.height].filter(Boolean);
 
-	console.log(release);
-
 	return release;
 }
 
-async function fetchLatest(channel, page = 1) {
-	const idRes = await unprint.request(channel.url, {
-		method: 'head',
-		followRedirects: 0,
-	});
+const profileIdRegex = /\/profile\/(\d+)/i;
 
-	const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1];
+async function getProfileId(channel) {
+	if (channel.url) {
+		const { hostname, pathname } = new URL(channel.url);
+
+		if (profileIdRegex.test(pathname)) { // full profile URL
+			const urlProfileId = new URL(channel.url).pathname.match(profileIdRegex)?.[1];
+
+			if (urlProfileId) {
+				return urlProfileId;
+			}
+		}
+
+		if (/[\w-]+.manyvids.com/.test(hostname)) { // short profile URL
+			const idRes = await unprint.request(channel.url, {
+				method: 'head',
+				followRedirects: 0,
+			});
+
+			const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
+
+			if (profileId) {
+				return profileId;
+			}
+		}
+
+		// sadly we can't reliably reconstruct short profile URL from the slug
+	}
+
+	return null;
+}
+
+async function fetchLatest(channel, page = 1) {
+	const profileId = await getProfileId(channel);
 
 	if (profileId) {
 		const res = await unprint.get(`https://api.manyvids.com/store/videos/${profileId}?sort=newest&limit=100&page=${page}`);
@@ -154,8 +184,6 @@ function scrapeProfile(data) {
 		profile.dateOfBirth = dateOfBirth;
 	}
 
-	console.log(profile);
-
 	return profile;
 }
 
@@ -166,7 +194,7 @@ async function getActorUrl(actor) {
 			followRedirects: 0,
 		});
 
-		const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1];
+		const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1];
 
 		if (profileId) {
 			return `https://api.manyvids.com/profile/profile/${profileId}`;