Updated default request limit config. Added type expect to Kink images to fix broken photos due malformed content type header.

This commit is contained in:
DebaucheryLibrarian 2024-10-24 03:18:27 +02:00
parent 87adc0472e
commit 01a8a24db6
4 changed files with 42 additions and 15 deletions

View File

@ -376,21 +376,28 @@ module.exports = {
interval: 50,
concurrency: 20,
},
'www.deeper.com': {
enable: false, // can be omitted to enable
'www.kink.com': {
interval: 1000,
concurrency: 1,
},
'westcoastproductions.com': {
interval: 100,
'www.brazzers.com': {
interval: 2000,
concurrency: 1,
},
'www.realitykings.com': {
interval: 2000,
concurrency: 1,
},
'www.analvids.com': { // seems to error at default rate
interval: 500,
concurrency: 1,
},
'images.mylfcdn.net': {
interval: 50,
concurrency: 1,
},
'www.analvids.com': { // seems to error at default rate
interval: 500,
'westcoastproductions.com': {
interval: 100,
concurrency: 1,
},
},

8
package-lock.json generated
View File

@ -89,7 +89,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.13.2",
"unprint": "^0.13.3",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",
@ -18312,9 +18312,9 @@
}
},
"node_modules/unprint": {
"version": "0.13.2",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz",
"integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==",
"version": "0.13.3",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.3.tgz",
"integrity": "sha512-HRpW+OdKmtW+cLnvLqYNVL2voH3aGvene8fxzAQzw2O0zPQrgv2iz5YivfQpxNyKsF+2jeUUma2ttWH8IttkHg==",
"dependencies": {
"axios": "^0.27.2",
"bottleneck": "^2.19.5",

View File

@ -148,7 +148,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.37",
"undici": "^5.28.1",
"unprint": "^0.13.2",
"unprint": "^0.13.3",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.1.3",
"video.js": "^8.6.1",

View File

@ -32,14 +32,24 @@ function scrapeAll(scenes, entity) {
stripQuery(poster).replace('_thumb', '_full'),
stripQuery(poster),
poster,
];
].filter(Boolean).map((src) => ({
src,
expectType: {
PNG: 'image/png',
},
}));
try {
release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
stripQuery(src).replace('_thumb', '_full'),
stripQuery(src),
src,
]);
].filter(Boolean).map((source) => ({
src: source,
expectType: {
PNG: 'image/png',
},
})));
} catch (error) {
// no photos
}
@ -79,12 +89,22 @@ function scrapeScene({ query }, url, entity) {
release.poster = [
stripQuery(poster),
poster,
];
].filter(Boolean).map((src) => ({
src,
expectType: {
PNG: 'image/png',
},
}));
release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [
src.fullPath,
src.thumbFullPath,
]);
].filter(Boolean).map((source) => ({
src: source,
expectType: {
PNG: 'image/png',
},
})));
release.trailer = [
...(data?.trailer?.sources?.map((source) => ({