diff --git a/package-lock.json b/package-lock.json index b60eb510..f8a2570d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -94,7 +94,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.18.27", + "unprint": "^0.18.29", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", @@ -20380,9 +20380,9 @@ } }, "node_modules/unprint": { - "version": "0.18.27", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.27.tgz", - "integrity": "sha512-XYLwX0vZPs9b+pK2h9P5dO3AWk8BUe41dnmbWSLU4qVGm8LYi5bBz7PP9wnaaPZJdAB8neaoZymFX4/WCjMR7g==", + "version": "0.18.29", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.18.29.tgz", + "integrity": "sha512-ZSlPLBf7kKW3X6y6ouner8loP9A0w+PEEFeR8eGdslH0P2LXML4rog0JtoJmbR2LoGqkAe0eb4Eeayolgvke4A==", "dependencies": { "bottleneck": "^2.19.5", "cookie": "^1.1.1", diff --git a/package.json b/package.json index df474d5c..0107a274 100755 --- a/package.json +++ b/package.json @@ -153,7 +153,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.37", "undici": "^5.28.1", - "unprint": "^0.18.27", + "unprint": "^0.18.29", "url-pattern": "^1.0.3", "v-tooltip": "^2.1.3", "video.js": "^8.6.1", diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 37dff94c..82af58eb 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -7843,7 +7843,7 @@ const sites = [ siteId: 321, native: true, }, - parent: 'milehighmedia', + parent: 'adultmobile', }, { slug: 'adultmobile', @@ -15117,6 +15117,9 @@ const sites = [ url: 'https://www.slayed.com', parent: 'vixen', tags: ['lesbian'], + parameters: { + useBrowser: true, + }, }, { slug: 'milfy', diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js index d468150d..7846e148 100755 --- a/src/scrapers/vixen.js +++ b/src/scrapers/vixen.js @@ -1,12 +1,9 @@ 'use strict'; /* eslint-disable newline-per-chained-call */ -const moment = require('moment'); const unprint = require('unprint'); const argv = require('../argv'); -const qu = require('../utils/qu'); -const http = require('../utils/http'); const genderMap = { F: 'female', @@ -60,7 +57,7 @@ function scrapeAll(scenes, channel) { release.url = `${channel.url}/videos/${data.slug}`; release.title = data.title; - release.date = qu.extractDate(data.releaseDate); + release.date = unprint.extractDate(data.releaseDate); release.actors = (data.modelsSlugged || data.models)?.map((model) => ({ name: model.name, url: model.slugged && `${channel.url}/models/${model.slugged}`, @@ -69,12 +66,27 @@ function scrapeAll(scenes, channel) { release.poster = curateSources(data.images.listing); release.teaser = curateSources(data.previews.listing, 'video/mp4'); - release.stars = data.rating; - return release; }); } +async function fetchLatest(site, page = 1) { + const url = `${site.url}/videos?page=${page}`; + const res = await unprint.get(url); + + if (res.ok) { + const data = res.context.query.json('#__NEXT_DATA__'); + + if (data?.props.pageProps.edges) { + return scrapeAll(data.props.pageProps.edges.map((edge) => edge.node), site); + } + + return []; + } + + return res.status; +} + function scrapeUpcoming(scenes, site) { return scenes.map((scene) => { if (!scene || scene.isPreReleasePeriod) { @@ -91,7 +103,7 @@ function scrapeUpcoming(scenes, site) { .map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`) .join(' '); - release.date = moment.utc(scene.releaseDate).toDate(); + release.date = unprint.extractDate(scene.releaseDate); release.datePrecision = 'minute'; release.actors = scene.models.map((model) => model.name); @@ -103,8 +115,97 @@ function scrapeUpcoming(scenes, site) { }).filter(Boolean); } +async function fetchUpcoming(channel) { + const query = ` + query getNextScene($site: Site!) { + nextScene: findNextReleaseVideo(input: { site: $site }) { + videoId + slug + isPreReleasePeriod + releaseDate + models { + name + __typename + } + images { + countdown { + ...ImageInfo + __typename + } + poster { + ...ImageInfo + __typename + } + __typename + } + previews { + countdown { + ...PreviewInfo + __typename + } + poster { + ...PreviewInfo + __typename + } + __typename + } + __typename + } + } + + fragment ImageInfo on Image { + src + placeholder + width + height + highdpi { + double + triple + __typename + } + webp { + src + placeholder + highdpi { + double + triple + __typename + } + __typename + } + } + + fragment PreviewInfo on Preview { + src + width + height + type + } + `; + + const res = await unprint.post(`${channel.url}/graphql`, { + operationName: 'getNextScene', + query, + variables: { + site: channel.slug.toUpperCase(), + }, + }, { + interface: 'request', + }); + + if (res.ok) { + if (res.data.data.nextScene) { + return scrapeUpcoming(res.data.data.nextScene, channel); + } + + return []; + } + + return res.status; +} + async function getTrailer(videoId, channel, url) { - const res = await http.post(`${channel.url}/graphql`, { + const res = await unprint.post(`${channel.url}/graphql`, { operationName: 'getToken', variables: { videoId, @@ -158,36 +259,37 @@ async function getTrailer(videoId, channel, url) { } `, }, { + interface: 'request', headers: { referer: url, origin: channel.url, }, }); - if (res.ok && res.body.data?.generateVideoToken) { + if (res.ok && res.data.data?.generateVideoToken) { return [ { - src: res.body.data.generateVideoToken.p2160?.token, + src: res.data.data.generateVideoToken.p2160?.token, quality: 2160, }, { - src: res.body.data.generateVideoToken.p1080?.token, + src: res.data.data.generateVideoToken.p1080?.token, quality: 1080, }, { - src: res.body.data.generateVideoToken.p720?.token, + src: res.data.data.generateVideoToken.p720?.token, quality: 720, }, { - src: res.body.data.generateVideoToken.p480?.token, + src: res.data.data.generateVideoToken.p480?.token, quality: 480, }, { - src: res.body.data.generateVideoToken.p360?.token, + src: res.data.data.generateVideoToken.p360?.token, quality: 360, }, { - src: res.body.data.generateVideoToken.p270?.token, + src: res.data.data.generateVideoToken.p270?.token, quality: 270, }, ]; @@ -204,12 +306,11 @@ async function scrapeScene(data, url, channel, options) { description: data.video.description, actors: data.video.models, director: data.video.directorNames, - duration: qu.durationToSeconds(data.video.runLength), - stars: data.video.rating, + duration: unprint.extractDuration(data.video.runLength), }; release.entryId = data.video.newId; - release.date = qu.extractDate(data.video.releaseDate); + release.date = unprint.extractDate(data.video.releaseDate); release.actors = data.video.modelsSlugged.map((model) => ({ name: model.name, @@ -266,7 +367,6 @@ async function scrapeSceneData(data, channel, options) { })); release.channel = data.site; - release.stars = data.rating; return release; } @@ -359,7 +459,7 @@ async function fetchGraphqlScene(release, channel) { } `; - const res = await http.post(`${channel.url}/graphql`, { + const res = await unprint.post(`${channel.url}/graphql`, { operationName: 'searchVideos', variables: { videoId: entryId, @@ -378,6 +478,7 @@ async function fetchGraphqlScene(release, channel) { } `, }, { + interface: 'request', headers: { referer: release.url, origin: channel.url, @@ -385,7 +486,7 @@ async function fetchGraphqlScene(release, channel) { }); if (res.ok) { - return res.body.data.video; + return res.data.data.video; } return null; @@ -398,14 +499,14 @@ async function fetchScene(url, channel, baseRelease, options) { return scrapeSceneData(graphqlData, channel, options); } - const session = qu.session(); - const res = await qu.get(url, null, null, { session }); + const res = await unprint.get(url, { + useBrowser: !!options.parameters?.useBrowser, + }); if (res.ok) { - const dataString = res.item.query.html('#__NEXT_DATA__'); - const data = dataString && JSON.parse(dataString); + const data = res.context.query.json('#__NEXT_DATA__'); - return scrapeScene(data.props.pageProps, url, channel, options, session); + return scrapeScene(data.props.pageProps, url, channel, options); } return res.status; @@ -436,113 +537,8 @@ async function scrapeProfile(data, channel) { return profile; } -async function fetchLatest(site, page = 1) { - const url = `${site.url}/videos?page=${page}`; - const res = await qu.get(url); - - if (res.ok) { - const dataString = res.item.query.html('#__NEXT_DATA__'); - const data = dataString && JSON.parse(dataString); - - if (data?.props.pageProps.edges) { - return scrapeAll(data.props.pageProps.edges.map((edge) => edge.node), site); - } - - return []; - } - - return res.status; -} - -async function fetchUpcoming(channel) { - const query = ` - query getNextScene($site: Site!) { - nextScene: findNextReleaseVideo(input: { site: $site }) { - videoId - slug - isPreReleasePeriod - releaseDate - models { - name - __typename - } - images { - countdown { - ...ImageInfo - __typename - } - poster { - ...ImageInfo - __typename - } - __typename - } - previews { - countdown { - ...PreviewInfo - __typename - } - poster { - ...PreviewInfo - __typename - } - __typename - } - __typename - } - } - - fragment ImageInfo on Image { - src - placeholder - width - height - highdpi { - double - triple - __typename - } - webp { - src - placeholder - highdpi { - double - triple - __typename - } - __typename - } - } - - fragment PreviewInfo on Preview { - src - width - height - type - } - `; - - const res = await http.post(`${channel.url}/graphql`, { - operationName: 'getNextScene', - query, - variables: { - site: channel.slug.toUpperCase(), - }, - }); - - if (res.ok) { - if (res.body.data.nextScene) { - return scrapeUpcoming(res.body.data.nextScene, channel); - } - - return []; - } - - return res.status; -} - async function fetchProfile(actor, { channel }) { - const res = await http.post(`${channel.url}/graphql`, { + const res = await unprint.post(`${channel.url}/graphql`, { operationName: 'searchModels', variables: { slug: actor.slug, @@ -605,14 +601,15 @@ async function fetchProfile(actor, { channel }) { ${imageFragment} `, }, { + interface: 'request', headers: { referer: channel.url, origin: channel.url, }, }); - if (res.ok && res.body.data?.model) { - return scrapeProfile(res.body.data, channel); + if (res.ok && res.data.data?.model) { + return scrapeProfile(res.data.data, channel); } return null; diff --git a/tests/profiles.js b/tests/profiles.js index c3e535ee..83da5ab0 100644 --- a/tests/profiles.js +++ b/tests/profiles.js @@ -256,8 +256,6 @@ const actors = [ { entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] }, ]; -// TODO: Brazzers, MetroHD, Blowpass, FameDigital, FantasyMassage, PrideStudios, Mamacitaz, Loveherfilms, Shelovesblack, Jerkaoke - const actorScrapers = scrapers.actors; const sources = argv.sources || null;