Updated default request limit config. Added type expect to Kink images to fix broken photos due malformed content type header.
This commit is contained in:
parent
87adc0472e
commit
01a8a24db6
|
@ -376,21 +376,28 @@ module.exports = {
|
|||
interval: 50,
|
||||
concurrency: 20,
|
||||
},
|
||||
'www.deeper.com': {
|
||||
enable: false, // can be omitted to enable
|
||||
'www.kink.com': {
|
||||
interval: 1000,
|
||||
concurrency: 1,
|
||||
},
|
||||
'westcoastproductions.com': {
|
||||
interval: 100,
|
||||
'www.brazzers.com': {
|
||||
interval: 2000,
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.realitykings.com': {
|
||||
interval: 2000,
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.analvids.com': { // seems to error at default rate
|
||||
interval: 500,
|
||||
concurrency: 1,
|
||||
},
|
||||
'images.mylfcdn.net': {
|
||||
interval: 50,
|
||||
concurrency: 1,
|
||||
},
|
||||
'www.analvids.com': { // seems to error at default rate
|
||||
interval: 500,
|
||||
'westcoastproductions.com': {
|
||||
interval: 100,
|
||||
concurrency: 1,
|
||||
},
|
||||
},
|
||||
|
|
|
@ -89,7 +89,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.13.2",
|
||||
"unprint": "^0.13.3",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
@ -18312,9 +18312,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/unprint": {
|
||||
"version": "0.13.2",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz",
|
||||
"integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==",
|
||||
"version": "0.13.3",
|
||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.3.tgz",
|
||||
"integrity": "sha512-HRpW+OdKmtW+cLnvLqYNVL2voH3aGvene8fxzAQzw2O0zPQrgv2iz5YivfQpxNyKsF+2jeUUma2ttWH8IttkHg==",
|
||||
"dependencies": {
|
||||
"axios": "^0.27.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
|
|
|
@ -148,7 +148,7 @@
|
|||
"tunnel": "0.0.6",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"undici": "^5.28.1",
|
||||
"unprint": "^0.13.2",
|
||||
"unprint": "^0.13.3",
|
||||
"url-pattern": "^1.0.3",
|
||||
"v-tooltip": "^2.1.3",
|
||||
"video.js": "^8.6.1",
|
||||
|
|
|
@ -32,14 +32,24 @@ function scrapeAll(scenes, entity) {
|
|||
stripQuery(poster).replace('_thumb', '_full'),
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
].filter(Boolean).map((src) => ({
|
||||
src,
|
||||
expectType: {
|
||||
PNG: 'image/png',
|
||||
},
|
||||
}));
|
||||
|
||||
try {
|
||||
release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
|
||||
stripQuery(src).replace('_thumb', '_full'),
|
||||
stripQuery(src),
|
||||
src,
|
||||
]);
|
||||
].filter(Boolean).map((source) => ({
|
||||
src: source,
|
||||
expectType: {
|
||||
PNG: 'image/png',
|
||||
},
|
||||
})));
|
||||
} catch (error) {
|
||||
// no photos
|
||||
}
|
||||
|
@ -79,12 +89,22 @@ function scrapeScene({ query }, url, entity) {
|
|||
release.poster = [
|
||||
stripQuery(poster),
|
||||
poster,
|
||||
];
|
||||
].filter(Boolean).map((src) => ({
|
||||
src,
|
||||
expectType: {
|
||||
PNG: 'image/png',
|
||||
},
|
||||
}));
|
||||
|
||||
release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [
|
||||
src.fullPath,
|
||||
src.thumbFullPath,
|
||||
]);
|
||||
].filter(Boolean).map((source) => ({
|
||||
src: source,
|
||||
expectType: {
|
||||
PNG: 'image/png',
|
||||
},
|
||||
})));
|
||||
|
||||
release.trailer = [
|
||||
...(data?.trailer?.sources?.map((source) => ({
|
||||
|
|
Loading…
Reference in New Issue