Updated default request limit config. Added type expect to Kink images to fix broken photos due malformed content type header.
This commit is contained in:
parent
87adc0472e
commit
01a8a24db6
|
@ -376,21 +376,28 @@ module.exports = {
|
||||||
interval: 50,
|
interval: 50,
|
||||||
concurrency: 20,
|
concurrency: 20,
|
||||||
},
|
},
|
||||||
'www.deeper.com': {
|
'www.kink.com': {
|
||||||
enable: false, // can be omitted to enable
|
|
||||||
interval: 1000,
|
interval: 1000,
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
},
|
},
|
||||||
'westcoastproductions.com': {
|
'www.brazzers.com': {
|
||||||
interval: 100,
|
interval: 2000,
|
||||||
|
concurrency: 1,
|
||||||
|
},
|
||||||
|
'www.realitykings.com': {
|
||||||
|
interval: 2000,
|
||||||
|
concurrency: 1,
|
||||||
|
},
|
||||||
|
'www.analvids.com': { // seems to error at default rate
|
||||||
|
interval: 500,
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
},
|
},
|
||||||
'images.mylfcdn.net': {
|
'images.mylfcdn.net': {
|
||||||
interval: 50,
|
interval: 50,
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
},
|
},
|
||||||
'www.analvids.com': { // seems to error at default rate
|
'westcoastproductions.com': {
|
||||||
interval: 500,
|
interval: 100,
|
||||||
concurrency: 1,
|
concurrency: 1,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
@ -89,7 +89,7 @@
|
||||||
"tunnel": "0.0.6",
|
"tunnel": "0.0.6",
|
||||||
"ua-parser-js": "^1.0.37",
|
"ua-parser-js": "^1.0.37",
|
||||||
"undici": "^5.28.1",
|
"undici": "^5.28.1",
|
||||||
"unprint": "^0.13.2",
|
"unprint": "^0.13.3",
|
||||||
"url-pattern": "^1.0.3",
|
"url-pattern": "^1.0.3",
|
||||||
"v-tooltip": "^2.1.3",
|
"v-tooltip": "^2.1.3",
|
||||||
"video.js": "^8.6.1",
|
"video.js": "^8.6.1",
|
||||||
|
@ -18312,9 +18312,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/unprint": {
|
"node_modules/unprint": {
|
||||||
"version": "0.13.2",
|
"version": "0.13.3",
|
||||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.2.tgz",
|
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.13.3.tgz",
|
||||||
"integrity": "sha512-AgSbo20TB8fWJUllEf8hwv0vlWzg/L4iKoy5Slh8as6+uR2ytJPODpCxxzgkli9vQwOyNJf1+CFgO5DZHLAjrw==",
|
"integrity": "sha512-HRpW+OdKmtW+cLnvLqYNVL2voH3aGvene8fxzAQzw2O0zPQrgv2iz5YivfQpxNyKsF+2jeUUma2ttWH8IttkHg==",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"axios": "^0.27.2",
|
"axios": "^0.27.2",
|
||||||
"bottleneck": "^2.19.5",
|
"bottleneck": "^2.19.5",
|
||||||
|
|
|
@ -148,7 +148,7 @@
|
||||||
"tunnel": "0.0.6",
|
"tunnel": "0.0.6",
|
||||||
"ua-parser-js": "^1.0.37",
|
"ua-parser-js": "^1.0.37",
|
||||||
"undici": "^5.28.1",
|
"undici": "^5.28.1",
|
||||||
"unprint": "^0.13.2",
|
"unprint": "^0.13.3",
|
||||||
"url-pattern": "^1.0.3",
|
"url-pattern": "^1.0.3",
|
||||||
"v-tooltip": "^2.1.3",
|
"v-tooltip": "^2.1.3",
|
||||||
"video.js": "^8.6.1",
|
"video.js": "^8.6.1",
|
||||||
|
|
|
@ -32,14 +32,24 @@ function scrapeAll(scenes, entity) {
|
||||||
stripQuery(poster).replace('_thumb', '_full'),
|
stripQuery(poster).replace('_thumb', '_full'),
|
||||||
stripQuery(poster),
|
stripQuery(poster),
|
||||||
poster,
|
poster,
|
||||||
];
|
].filter(Boolean).map((src) => ({
|
||||||
|
src,
|
||||||
|
expectType: {
|
||||||
|
PNG: 'image/png',
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
|
release.photos = JSON.parse(query.attribute('.ratio-thumbnail img', 'data-cycle')).map((src) => [
|
||||||
stripQuery(src).replace('_thumb', '_full'),
|
stripQuery(src).replace('_thumb', '_full'),
|
||||||
stripQuery(src),
|
stripQuery(src),
|
||||||
src,
|
src,
|
||||||
]);
|
].filter(Boolean).map((source) => ({
|
||||||
|
src: source,
|
||||||
|
expectType: {
|
||||||
|
PNG: 'image/png',
|
||||||
|
},
|
||||||
|
})));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// no photos
|
// no photos
|
||||||
}
|
}
|
||||||
|
@ -79,12 +89,22 @@ function scrapeScene({ query }, url, entity) {
|
||||||
release.poster = [
|
release.poster = [
|
||||||
stripQuery(poster),
|
stripQuery(poster),
|
||||||
poster,
|
poster,
|
||||||
];
|
].filter(Boolean).map((src) => ({
|
||||||
|
src,
|
||||||
|
expectType: {
|
||||||
|
PNG: 'image/png',
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [
|
release.photos = query.json('#galleryImagesContainer', { attribute: 'data-images' })?.map((src) => [
|
||||||
src.fullPath,
|
src.fullPath,
|
||||||
src.thumbFullPath,
|
src.thumbFullPath,
|
||||||
]);
|
].filter(Boolean).map((source) => ({
|
||||||
|
src: source,
|
||||||
|
expectType: {
|
||||||
|
PNG: 'image/png',
|
||||||
|
},
|
||||||
|
})));
|
||||||
|
|
||||||
release.trailer = [
|
release.trailer = [
|
||||||
...(data?.trailer?.sources?.map((source) => ({
|
...(data?.trailer?.sources?.map((source) => ({
|
||||||
|
|
Loading…
Reference in New Issue