From 01a8a24db67be279903d0fcaed1d5424a20f9aee Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Thu, 24 Oct 2024 03:18:27 +0200 Subject: [PATCH] Updated default request limit config. Added type expect to Kink images to fix broken photos due malformed content type header. --- config/default.js | 19 +++++++++++++------ package-lock.json | 8 ++++---- package.json | 2 +- src/scrapers/kink.js | 28 ++++++++++++++++++++++++---- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/config/default.js b/config/default.js index 918e4a9b..a8d98985 100755 --- a/config/default.js +++ b/config/default.js @@ -376,21 +376,28 @@ module.exports = { interval: 50, concurrency: 20, }, - 'www.deeper.com': { - enable: false, // can be omitted to enable + 'www.kink.com': { interval: 1000, concurrency: 1, }, - 'westcoastproductions.com': { - interval: 100, + 'www.brazzers.com': { + interval: 2000, + concurrency: 1, + }, + 'www.realitykings.com': { + interval: 2000, + concurrency: 1, + }, + 'www.analvids.com': { // seems to error at default rate + interval: 500, concurrency: 1, }, 'images.mylfcdn.net': { interval: 50, concurrency: 1, }, - 'www.analvids.com': { // seems to error at default rate - interval: 500, + 'westcoastproductions.com': { + interval: 100, concurrency: 1, }, }, diff --git a/package-lock.json b/package-lock.json index caa910d2..f5828eb1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,7 +89,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.13.2", + "unprint": "^0.13.3", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", @@ -18312,9 +18312,9 @@ } }, "node_modules/unprint": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz", - "integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==", + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.3.tgz", + "integrity": "sha512-HRpW+OdKmtW+cLnvLqYNVL2voH3aGvene8fxzAQzw2O0zPQrgv2iz5YivfQpxNyKsF+2jeUUma2ttWH8IttkHg==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index ae6da35c..f3152a17 100755 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.13.2", + "unprint": "^0.13.3", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", diff --git a/src/scrapers/kink.js b/src/scrapers/kink.js index 692ab4a0..79bf7c79 100755 --- a/src/scrapers/kink.js +++ b/src/scrapers/kink.js @@ -32,14 +32,24 @@ function scrapeAll(scenes, entity) { stripQuery(poster).replace('_thumb', '_full'), stripQuery(poster), poster, - ]; + ].filter(Boolean).map((src) => ({ + src, + expectType: { + PNG: 'image/png', + }, + })); try { release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [ stripQuery(src).replace('_thumb', '_full'), stripQuery(src), src, - ]); + ].filter(Boolean).map((source) => ({ + src: source, + expectType: { + PNG: 'image/png', + }, + }))); } catch (error) { // no photos } @@ -79,12 +89,22 @@ function scrapeScene({ query }, url, entity) { release.poster = [ stripQuery(poster), poster, - ]; + ].filter(Boolean).map((src) => ({ + src, + expectType: { + PNG: 'image/png', + }, + })); release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [ src.fullPath, src.thumbFullPath, - ]); + ].filter(Boolean).map((source) => ({ + src: source, + expectType: { + PNG: 'image/png', + }, + }))); release.trailer = [ ...(data?.trailer?.sources?.map((source) => ({