diff --git a/package-lock.json b/package-lock.json index 8427e05a..de6f5a75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -93,7 +93,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.18.1", + "unprint": "^0.18.4", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", @@ -20340,9 +20340,9 @@ } }, "node_modules/unprint": { - "version": "0.18.1", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.1.tgz", - "integrity": "sha512-lL7aIQfVoVY3oC69WM6GOUabwfJkMQxM4XfIT5EA21CsisJassWU3sX2ZQqIrsX4NujOeDVgzU2qYFJ/QWpoqQ==", + "version": "0.18.4", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.4.tgz", + "integrity": "sha512-npOTOoKrnxv+Zs7AFz3pXfyV7VftRr0ULAhCbnodPNvBoymde9JnVuRGJ6JL6q6RfH0wJgO5jpsT1ar844733g==", "dependencies": { "bottleneck": "^2.19.5", "cookie": "^1.1.1", diff --git a/package.json b/package.json index 873ead1e..16f0cd56 100755 --- a/package.json +++ b/package.json @@ -152,7 +152,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.18.1", + "unprint": "^0.18.4", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", diff --git a/src/app.js b/src/app.js index d9568681..7bd78ad5 100755 --- a/src/app.js +++ b/src/app.js @@ -32,13 +32,9 @@ let done = false; unprint.options({ timeout: argv.requestTimeout, - headers: { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', - }, - context: { - // browser requests - userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', - }, + userAgent: 'traxxx', + browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + apiUserAgent: 'traxxx', limits: { ...config.limits, default: { diff --git a/src/scrapers/bellesa.js b/src/scrapers/bellesa.js index b1ba5131..cc867c75 100644 --- a/src/scrapers/bellesa.js +++ b/src/scrapers/bellesa.js @@ -65,11 +65,11 @@ async function fetchLatest(channel, page = 1, { parameters }) { const provider = new URL(channel.url).searchParams.get('providers'); const res = await unprint.get(`${parameters.api}/videos?filter[provider]=${provider}&limit=24&sources=${parameters.source}&page=${page}`, { + interface: 'request', headers: { // currently only required for Bellesa Plus Referer: `${new URL(channel.url).origin}/videos`, Cookie: 'bellesa_agegate=true', - 'User-Agent': null, }, }); diff --git a/src/scrapers/nubiles.js b/src/scrapers/nubiles.js index affd8f43..73fe81ee 100755 --- a/src/scrapers/nubiles.js +++ b/src/scrapers/nubiles.js @@ -21,7 +21,10 @@ function stripQuery(link) { } async function getPhotos(albumUrl) { - const res = await unprint.get(albumUrl, { selectAll: '.photo-thumb' }); + const res = await unprint.get(albumUrl, { + selectAll: '.photo-thumb', + interface: 'request', + }); return res.ok ? res.context.map(({ query }) => unprint.prefixUrl(query.element('source').srcset)) @@ -65,8 +68,12 @@ function scrapeAll(scenes, entity) { } async function fetchLatest(site, page = 1) { - const url = `${site.url}/video/gallery/${(page - 1) * 12}`; - const res = await unprint.get(url, { selectAll: '.content-grid-item' }); + const url = `${site.url}/video/gallery/${((page - 1) * 12) || ''}`; + + const res = await unprint.get(url, { + interface: 'request', + selectAll: '.content-grid-item', + }); if (res.ok) { return scrapeAll(res.context, site); @@ -78,7 +85,11 @@ async function fetchLatest(site, page = 1) { async function fetchUpcoming(site) { if (site.parameters?.upcoming) { const url = `${site.url}/video/upcoming`; - const res = await unprint.get(url, { selectAll: '.content-grid-item' }); + + const res = await unprint.get(url, { + selectAll: '.content-grid-item', + interface: 'request', + }); if (res.ok) { return scrapeAll(res.context, site); @@ -121,12 +132,24 @@ async function scrapeScene({ query }, { url, entity, include }) { const albumLink = query.url('.content-pane-related-links a[href*="gallery"]'); if (albumLink && include.photos) { - release.photos = await getPhotos(albumLink); + release.photos = await getPhotos(unprint.prefixUrl(albumLink, new URL(entity.url).origin)); } return release; } +async function fetchScene(url, entity, _baseRelease, include) { + const res = await unprint.get(url, { + interface: 'request', + }); + + if (res.ok) { + return scrapeScene(res.context, { url, entity, include }); + } + + return res.status; +} + function scrapeProfile({ query }, avatar) { const profile = {}; @@ -161,7 +184,10 @@ async function findModel(actor, entity) { const origin = slugUrlMap[entity.slug] || entity.url; const url = `${origin}/model/alpha/${firstLetter}`; - const resModels = await unprint.get(url); + + const resModels = await unprint.get(url, { + interface: 'request', + }); if (!resModels.ok) { return resModels.status; @@ -191,7 +217,9 @@ async function fetchProfile(actor, { entity }) { const model = await findModel(actor, entity); if (model) { - const resModel = await unprint.get(model.url); + const resModel = await unprint.get(model.url, { + interface: 'request', + }); if (resModel.ok) { return scrapeProfile(resModel.context, model.avatar); @@ -207,5 +235,5 @@ module.exports = { fetchLatest, fetchUpcoming, fetchProfile, - scrapeScene, + fetchScene, };