diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 0aa258b6..a8784bbd 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -700,7 +700,9 @@ const networks = [ parent: 'chargedmedia', tags: ['gay'], parameters: { - endpoint: 'sau-elastic-00gy5fg5ra', + // endpoint: 'sau-elastic-00gy5fg5ra', + fullEndpoint: 'sau_network', + modelPrefix: 'model_', }, }, { diff --git a/src/scrapers/score.js b/src/scrapers/score.js index b3949dc3..40e53687 100755 --- a/src/scrapers/score.js +++ b/src/scrapers/score.js @@ -27,6 +27,7 @@ function scrapeAll(scenes, channel, parameters) { const poster = query.img('.item-img img'); const url = stripQuery(query.url('a.i-title, .item-img a')); + const { pathname, hostname } = new URL(url); release.title = query.content('a.i-title, h2.i-title'); release.duration = query.duration('.time-ol'); @@ -44,7 +45,7 @@ function scrapeAll(scenes, channel, parameters) { release.actors = query.content('.i-model').split(',').map((actor) => actor.trim()); - if (url.includes('join.') || url.includes('/join')) { + if (hostname.includes('join.') || pathname.includes('/join') || pathname.length <= 1) { // no link available, attempt to reconstruct from poster URL const entryId = poster?.match(/posting_(\d+)/)?.[1]; @@ -58,7 +59,7 @@ function scrapeAll(scenes, channel, parameters) { } } else { release.url = url; - release.entryId = new URL(release.url).pathname.match(/\/(\d+)\/?$/)[1]; + release.entryId = pathname.match(/\/(\d+)\/?$/)[1]; } if (poster) { diff --git a/src/scrapers/teamskeet.js b/src/scrapers/teamskeet.js index 5b504d7c..fda02142 100755 --- a/src/scrapers/teamskeet.js +++ b/src/scrapers/teamskeet.js @@ -128,7 +128,7 @@ function scrapeProfile(actor, entity, parameters) { } async function fetchLatest(channel, page = 1, { parameters }) { - const res = await http.get(`https://tours-store.psmcdn.net/${parameters.endpoint}-videoscontent/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); + const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_search?q=site.seo.seoSlug:"${parameters.id}"&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); if (res.ok) { return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); @@ -157,7 +157,7 @@ async function fetchLatestOrganic(channel, page, context) { } async function fetchLatestSearch(channel, page = 1, { parameters }) { - const res = await http.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:%22${parameters.id}%22%20AND%20type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); + const res = await http.get(`https://tours-store.psmcdn.net/${parameters.fullEndpoint || parameters.endpoint}/_search?q=(site.seo.seoSlug:%22${parameters.id}%22%20AND%20type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`); if (res.ok) { return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters); @@ -177,7 +177,7 @@ async function fetchScene(url, channel, baseScene, { parameters }) { const res = await http.get({ organic: `https://store.psmcdn.net/${parameters.endpoint}/moviesContent/${sceneSlug}.json`, search: `https://tours-store.psmcdn.net/ts_network/_search/?q=(id:${sceneSlug})&size=1`, - undefined: `https://tours-store.psmcdn.net/${parameters.endpoint}-videoscontent/_doc/${sceneSlug}`, + undefined: `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-videoscontent`}/_doc/${sceneSlug}`, }[parameters.layout]); if (res.ok && res.body.found) { @@ -195,7 +195,7 @@ async function fetchProfile(baseActor, { entity, parameters }) { // const url = format(parameters.profiles, { slug: baseActor.slug }); const url = parameters.layout === 'organic' ? `https://store.psmcdn.net/${parameters.endpoint}/modelsContent/${baseActor.slug}.json` - : `https://tours-store.psmcdn.net/${parameters.endpoint}-modelscontent/_doc/${baseActor.slug}`; + : `https://tours-store.psmcdn.net/${parameters.fullEndpoint || `${parameters.endpoint}-modelscontent`}/_doc/${parameters.modelPrefix || ''}${baseActor.slug}`; const res = await qu.get(url); diff --git a/tests/profiles.js b/tests/profiles.js index 756630a6..e1917e2d 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -25,7 +25,8 @@ const actors = [ { entity: 'wifey', name: 'Danielle Renae', fields: ['gender', 'avatar', 'description'] }, // teamskeet { entity: 'teamskeet', name: 'Abella Danger', fields: ['description', 'avatar', 'measurements', 'birthPlace', 'nationality', 'ethnicity', 'height', 'weight', 'hairColor', 'hasPiercings'] }, - { entity: 'teamskeet', name: 'Kali Roses', fields: ['description', 'avatar', 'measurements', 'nationality', 'ethnicity', 'hairColor', 'hasPiercings', 'hasTattoos'] }, // tattoos + { entity: 'mylf', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'measurements', 'nationality', 'hairColor', 'hasPiercings', 'hasTattoos'] }, + { entity: 'sayuncle', name: 'Greg McKeon', fields: ['avatar', 'description'] }, // mike adriano { entity: 'trueanal', name: 'Brenna McKenna', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor', 'hasTattoos'] }, { entity: 'analonly', name: 'Lilith Grace', fields: ['avatar', 'gender', 'description', 'dateOfBirth', 'birthPlace', 'measurements', 'eyes', 'weight', 'height', 'hairColor'] },