diff --git a/seeds/00_tags.js b/seeds/00_tags.js index b1bb0a52..6ef89fe2 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -762,6 +762,11 @@ const tags = [ slug: 'high-heels', group: 'clothing', }, + { + name: 'hotel', + slug: 'hotel', + group: 'location', + }, { name: 'humiliation', slug: 'humiliation', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 1cd41c4c..7feea4f4 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -7698,6 +7698,7 @@ const sites = [ url: 'https://nicoledoshi.manyvids.com', hasLogo: false, parent: 'manyvids', + delete: true, }, // MARISKA X { diff --git a/seeds/09_manyvids.js b/seeds/09_manyvids.js new file mode 100644 index 00000000..ac7a7acb --- /dev/null +++ b/seeds/09_manyvids.js @@ -0,0 +1,56 @@ +'use strict'; + +const unprint = require('unprint'); +const slugify = require('../src/utils/slugify'); + +const limit = Number(process.env.MV_LIMIT) || 1000; + +async function fetchCreators(page = 1, acc = []) { + console.log(`Fetching creators ${acc.length}/${limit}`); + + // size seems to be capped at 70 + const res = await unprint.get(`https://api.manyvids.com/search/creators/list?contentPref=1,2,3&sort=top&size=50&from=${(page - 1) * 50}`); + + if (res.ok && res.data?.creators) { + const creators = acc.concat(res.data.creators.map((creator) => ({ + name: creator.stageName, + slug: `mv.${slugify(creator.slug, '')}`, + url: `https://www.manyvids.com/Profile/${creator.id}/${creator.slug}/`, + emblem: creator.avatar?.url || null, + }))); + + if (creators.length >= limit) { + return creators; + } + + return fetchCreators(page + 1, creators); + } + + return acc; +} + +exports.seed = async function (knex) { + const channels = await fetchCreators(); + + const manyvidsNetwork = await knex('entities') + .where('slug', 'manyvids') + .where('type', 'network') + .first(); + + if (!manyvidsNetwork) { + throw new Error('ManyVids found, did the network migration run?'); + } + + await knex('entities') + .insert(channels.map((channel) => ({ + name: channel.name, + slug: channel.slug, + url: channel.url, + parent_id: manyvidsNetwork.id, + has_logo: false, + }))) + .onConflict(['slug', 'type']) + .merge(['name', 'url']); + + console.log(`Done! ${channels.length} ManyVids creators added as channels`); +}; diff --git a/src/scrapers/manyvids.js b/src/scrapers/manyvids.js index f36268f6..7d1a6a6b 100755 --- a/src/scrapers/manyvids.js +++ b/src/scrapers/manyvids.js @@ -26,6 +26,7 @@ async function getTrailer(videoId) { return null; } +/* function getModelUrl(modelData) { const slug = modelData.slug || modelData.profileUrl?.match(/\/profile\/\d+\/(\w+)/i)?.[1] || null; @@ -35,6 +36,7 @@ function getModelUrl(modelData) { return null; } +*/ async function scrapeScene(data, context) { const release = {}; @@ -61,6 +63,7 @@ async function scrapeScene(data, context) { release.date = new Date(data.launchDate); release.duration = unprint.extractDuration(data.duration || data.videoDuration); + /* many channels operate more like a studio than a performer, so the channel information doesn't represent a person const model = data.creator || data.model; release.actors = [{ @@ -69,6 +72,7 @@ async function scrapeScene(data, context) { avatar: model.avatar?.url || model.avatar, url: getModelUrl(model), }]; + */ release.tags = [ data.is3D && '3d', @@ -86,18 +90,44 @@ async function scrapeScene(data, context) { release.qualities = [data.height].filter(Boolean); - console.log(release); - return release; } -async function fetchLatest(channel, page = 1) { - const idRes = await unprint.request(channel.url, { - method: 'head', - followRedirects: 0, - }); +const profileIdRegex = /\/profile\/(\d+)/i; - const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1]; +async function getProfileId(channel) { + if (channel.url) { + const { hostname, pathname } = new URL(channel.url); + + if (profileIdRegex.test(pathname)) { // full profile URL + const urlProfileId = new URL(channel.url).pathname.match(profileIdRegex)?.[1]; + + if (urlProfileId) { + return urlProfileId; + } + } + + if (/[\w-]+.manyvids.com/.test(hostname)) { // short profile URL + const idRes = await unprint.request(channel.url, { + method: 'head', + followRedirects: 0, + }); + + const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1]; + + if (profileId) { + return profileId; + } + } + + // sadly we can't reliably reconstruct short profile URL from the slug + } + + return null; +} + +async function fetchLatest(channel, page = 1) { + const profileId = await getProfileId(channel); if (profileId) { const res = await unprint.get(`https://api.manyvids.com/store/videos/${profileId}?sort=newest&limit=100&page=${page}`); @@ -154,8 +184,6 @@ function scrapeProfile(data) { profile.dateOfBirth = dateOfBirth; } - console.log(profile); - return profile; } @@ -166,7 +194,7 @@ async function getActorUrl(actor) { followRedirects: 0, }); - const profileId = idRes.headers.get('location')?.match(/\/profile\/(\d+)/i)?.[1]; + const profileId = idRes.headers.get('location')?.match(profileIdRegex)?.[1]; if (profileId) { return `https://api.manyvids.com/profile/profile/${profileId}`;