Updated default request limit config. Added type expect to Kink images to fix broken photos due malformed content type header.

This commit is contained in:
DebaucheryLibrarian 2024-10-24 03:18:27 +02:00
parent 87adc0472e
commit 01a8a24db6
4 changed files with 42 additions and 15 deletions

View File

@ -376,21 +376,28 @@ module.exports = {
interval: 50, interval: 50,
concurrency: 20, concurrency: 20,
}, },
'www.deeper.com': { 'www.kink.com': {
enable: false, // can be omitted to enable
interval: 1000, interval: 1000,
concurrency: 1, concurrency: 1,
}, },
'westcoastproductions.com': { 'www.brazzers.com': {
interval: 100, interval: 2000,
concurrency: 1,
},
'www.realitykings.com': {
interval: 2000,
concurrency: 1,
},
'www.analvids.com': { // seems to error at default rate
interval: 500,
concurrency: 1, concurrency: 1,
}, },
'images.mylfcdn.net': { 'images.mylfcdn.net': {
interval: 50, interval: 50,
concurrency: 1, concurrency: 1,
}, },
'www.analvids.com': { // seems to error at default rate 'westcoastproductions.com': {
interval: 500, interval: 100,
concurrency: 1, concurrency: 1,
}, },
}, },

8
package-lock.json generated
View File

@ -89,7 +89,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.13.2", "unprint": "^0.13.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",
@ -18312,9 +18312,9 @@
} }
}, },
"node_modules/unprint": { "node_modules/unprint": {
"version": "0.13.2", "version": "0.13.3",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.3.tgz",
"integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==", "integrity": "sha512-HRpW+OdKmtW+cLnvLqYNVL2voH3aGvene8fxzAQzw2O0zPQrgv2iz5YivfQpxNyKsF+2jeUUma2ttWH8IttkHg==",
"dependencies": { "dependencies": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",

View File

@ -148,7 +148,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.37", "ua-parser-js": "^1.0.37",
"undici": "^5.28.1", "undici": "^5.28.1",
"unprint": "^0.13.2", "unprint": "^0.13.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3", "v-tooltip": "^2.1.3",
"video.js": "^8.6.1", "video.js": "^8.6.1",

View File

@ -32,14 +32,24 @@ function scrapeAll(scenes, entity) {
stripQuery(poster).replace('_thumb', '_full'), stripQuery(poster).replace('_thumb', '_full'),
stripQuery(poster), stripQuery(poster),
poster, poster,
]; ].filter(Boolean).map((src) => ({
src,
expectType: {
PNG: 'image/png',
},
}));
try { try {
release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [ release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
stripQuery(src).replace('_thumb', '_full'), stripQuery(src).replace('_thumb', '_full'),
stripQuery(src), stripQuery(src),
src, src,
]); ].filter(Boolean).map((source) => ({
src: source,
expectType: {
PNG: 'image/png',
},
})));
} catch (error) { } catch (error) {
// no photos // no photos
} }
@ -79,12 +89,22 @@ function scrapeScene({ query }, url, entity) {
release.poster = [ release.poster = [
stripQuery(poster), stripQuery(poster),
poster, poster,
]; ].filter(Boolean).map((src) => ({
src,
expectType: {
PNG: 'image/png',
},
}));
release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [ release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [
src.fullPath, src.fullPath,
src.thumbFullPath, src.thumbFullPath,
]); ].filter(Boolean).map((source) => ({
src: source,
expectType: {
PNG: 'image/png',
},
})));
release.trailer = [ release.trailer = [
...(data?.trailer?.sources?.map((source) => ({ ...(data?.trailer?.sources?.map((source) => ({