Updated Kink scraper, added age gate cookie, fixed date query and trailer URL.
This commit is contained in:
8
package-lock.json
generated
8
package-lock.json
generated
@@ -95,7 +95,7 @@
|
|||||||
"tunnel": "0.0.6",
|
"tunnel": "0.0.6",
|
||||||
"ua-parser-js": "^1.0.37",
|
"ua-parser-js": "^1.0.37",
|
||||||
"undici": "^7.24.7",
|
"undici": "^7.24.7",
|
||||||
"unprint": "^0.19.13",
|
"unprint": "^0.19.18",
|
||||||
"url-pattern": "^1.0.3",
|
"url-pattern": "^1.0.3",
|
||||||
"v-tooltip": "^2.1.3",
|
"v-tooltip": "^2.1.3",
|
||||||
"video.js": "^8.6.1",
|
"video.js": "^8.6.1",
|
||||||
@@ -20844,9 +20844,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/unprint": {
|
"node_modules/unprint": {
|
||||||
"version": "0.19.13",
|
"version": "0.19.18",
|
||||||
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.13.tgz",
|
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.19.18.tgz",
|
||||||
"integrity": "sha512-HPNCQn2CziiGeK0JSZg/5E+G2prHme+8lDojxd16wUwSQ0mgW4nZq4LOuVMIRRAFm1M1nkju0oMIdsj4uRFASw==",
|
"integrity": "sha512-8opK8pRRJ5GY+gBBeIieD5Hr1D3ldWsiyvG4I+60Rjhx2EHkyhYL/okl6fOpiwvC4aoB+G69wzDipwSGWX5t1w==",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"bottleneck": "^2.19.5",
|
"bottleneck": "^2.19.5",
|
||||||
"cookie": "^1.1.1",
|
"cookie": "^1.1.1",
|
||||||
|
|||||||
@@ -154,7 +154,7 @@
|
|||||||
"tunnel": "0.0.6",
|
"tunnel": "0.0.6",
|
||||||
"ua-parser-js": "^1.0.37",
|
"ua-parser-js": "^1.0.37",
|
||||||
"undici": "^7.24.7",
|
"undici": "^7.24.7",
|
||||||
"unprint": "^0.19.13",
|
"unprint": "^0.19.18",
|
||||||
"url-pattern": "^1.0.3",
|
"url-pattern": "^1.0.3",
|
||||||
"v-tooltip": "^2.1.3",
|
"v-tooltip": "^2.1.3",
|
||||||
"video.js": "^8.6.1",
|
"video.js": "^8.6.1",
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ function scrapeAll(scenes, entity) {
|
|||||||
release.entryId = release.shootId;
|
release.entryId = release.shootId;
|
||||||
|
|
||||||
release.title = query.content('.card-body a[href*="/shoot"]').trim();
|
release.title = query.content('.card-body a[href*="/shoot"]').trim();
|
||||||
release.date = query.date('small > span', 'MMM D, YYYY');
|
release.date = query.date('.small > span', 'MMM D, YYYY');
|
||||||
|
|
||||||
release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
|
release.actors = query.all('a[href*="/model"]').map((actorEl) => ({
|
||||||
name: unprint.query.content(actorEl),
|
name: unprint.query.content(actorEl),
|
||||||
@@ -54,7 +54,11 @@ function scrapeAll(scenes, entity) {
|
|||||||
// no photos
|
// no photos
|
||||||
}
|
}
|
||||||
|
|
||||||
release.trailer = `https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`;
|
release.trailer = [
|
||||||
|
query.video('.ratio-thumbnail img', { attribute: 'data-trailer-url' }),
|
||||||
|
`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
|
||||||
|
`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
|
||||||
|
];
|
||||||
|
|
||||||
release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
|
release.channel = slugify(query.content('.shoot-thumbnail-footer a[href*="/channel"]'), '');
|
||||||
release.rating = query.number('.thumb-up') / 10;
|
release.rating = query.number('.thumb-up') / 10;
|
||||||
@@ -64,10 +68,13 @@ function scrapeAll(scenes, entity) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(channel, page = 1) {
|
async function fetchLatest(channel, page = 1) {
|
||||||
const url = `${channel.parent.url}/search?type=shoots&channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
|
const url = `${channel.parent.url}/shoots?channelIds=${channel.parameters?.slug || channel.slug}&sort=published&page=${page}`;
|
||||||
|
|
||||||
const res = await unprint.browser(url, {
|
const res = await unprint.browser(url, {
|
||||||
selectAll: '.container .card',
|
selectAll: '.container .card',
|
||||||
|
cookies: {
|
||||||
|
age_gate_accepted: '1',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 200) {
|
if (res.status === 200) {
|
||||||
@@ -129,6 +136,7 @@ function scrapeScene({ query }, url, entity) {
|
|||||||
src: source.url,
|
src: source.url,
|
||||||
quality: source.resolution,
|
quality: source.resolution,
|
||||||
})) || []),
|
})) || []),
|
||||||
|
`https://cdnp.kink.com/v2/imagedb/shoots/${release.entryId}/public/trailer/${release.entryId}_trailer_high.mp4`,
|
||||||
`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
|
`https://cdnp.kink.com/imagedb/${release.entryId}/trailer/${release.entryId}_trailer_high.mp4`,
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -143,7 +151,11 @@ function scrapeScene({ query }, url, entity) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function fetchScene(url, channel) {
|
async function fetchScene(url, channel) {
|
||||||
const res = await unprint.browser(url);
|
const res = await unprint.browser(url, {
|
||||||
|
cookies: {
|
||||||
|
age_gate_accepted: '1',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
if (res.status === 200) {
|
if (res.status === 200) {
|
||||||
const scene = scrapeScene(res.context, url, channel);
|
const scene = scrapeScene(res.context, url, channel);
|
||||||
@@ -197,6 +209,9 @@ async function getActorUrl({ name: actorName, url }, networkUrl) {
|
|||||||
// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
|
// const searchRes = await tab.goto(`${networkUrl}/search?type=performers&q=${actorName}`);
|
||||||
const res = await unprint.get(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`, {
|
const res = await unprint.get(`https://www.kink.com/api/v2/search/suggestions/performers?term=${actorName}`, {
|
||||||
interface: 'request',
|
interface: 'request',
|
||||||
|
cookies: {
|
||||||
|
age_gate_accepted: '1',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 200) {
|
if (res.status === 200) {
|
||||||
@@ -217,7 +232,11 @@ async function fetchProfile(actor, entity) {
|
|||||||
const actorUrl = await getActorUrl(actor, networkUrl);
|
const actorUrl = await getActorUrl(actor, networkUrl);
|
||||||
|
|
||||||
if (actorUrl) {
|
if (actorUrl) {
|
||||||
const actorRes = await unprint.browser(actorUrl);
|
const actorRes = await unprint.browser(actorUrl, {
|
||||||
|
cookies: {
|
||||||
|
age_gate_accepted: '1',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
if (actorRes.status === 200) {
|
if (actorRes.status === 200) {
|
||||||
return scrapeProfile(actorRes.context, actorUrl);
|
return scrapeProfile(actorRes.context, actorUrl);
|
||||||
|
|||||||
Reference in New Issue
Block a user