diff --git a/package-lock.json b/package-lock.json index f875231ef..b8c9cb10d 100755 --- a/package-lock.json +++ b/package-lock.json @@ -80,7 +80,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.10.0", + "unprint": "^0.10.1", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", @@ -17538,9 +17538,9 @@ } }, "node_modules/unprint": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.0.tgz", - "integrity": "sha512-HQ73xcI7wzWeK3j3EQ0wWDe1dpMmeKY4LpFg9bNPD5G8ZyqYm5hYx1KGu6J9NTV/T5tOW4Ec0zyHoCRT2Msimg==", + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", + "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", @@ -32378,9 +32378,9 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "unprint": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.0.tgz", - "integrity": "sha512-HQ73xcI7wzWeK3j3EQ0wWDe1dpMmeKY4LpFg9bNPD5G8ZyqYm5hYx1KGu6J9NTV/T5tOW4Ec0zyHoCRT2Msimg==", + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", + "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", "requires": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index 0f2ededb1..cb1aa883e 100755 --- a/package.json +++ b/package.json @@ -139,7 +139,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.10.0", + "unprint": "^0.10.1", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", diff --git a/public/img/logos/filthykings/fillupmymom.png b/public/img/logos/filthykings/fillupmymom.png new file mode 100644 index 000000000..fbb5f6135 Binary files /dev/null and b/public/img/logos/filthykings/fillupmymom.png differ diff --git a/public/img/logos/filthykings/filthyblowjobs.png b/public/img/logos/filthykings/filthyblowjobs.png new file mode 100644 index 000000000..a24153c4f Binary files /dev/null and b/public/img/logos/filthykings/filthyblowjobs.png differ diff --git a/public/img/logos/filthykings/filthymassage.png b/public/img/logos/filthykings/filthymassage.png new file mode 100644 index 000000000..fe6227ed9 Binary files /dev/null and b/public/img/logos/filthykings/filthymassage.png differ diff --git a/public/img/logos/filthykings/filthynewbies.png b/public/img/logos/filthykings/filthynewbies.png new file mode 100644 index 000000000..e52fe0a73 Binary files /dev/null and b/public/img/logos/filthykings/filthynewbies.png differ diff --git a/public/img/logos/filthykings/filthypov.png b/public/img/logos/filthykings/filthypov.png new file mode 100644 index 000000000..7c11f7d84 Binary files /dev/null and b/public/img/logos/filthykings/filthypov.png differ diff --git a/public/img/logos/filthykings/filthytaboo.png b/public/img/logos/filthykings/filthytaboo.png new file mode 100644 index 000000000..2ffcae853 Binary files /dev/null and b/public/img/logos/filthykings/filthytaboo.png differ diff --git a/public/img/logos/filthykings/fkbts.png b/public/img/logos/filthykings/fkbts.png new file mode 100644 index 000000000..d38ff1897 Binary files /dev/null and b/public/img/logos/filthykings/fkbts.png differ diff --git a/public/img/logos/filthykings/hotgirlsraw.png b/public/img/logos/filthykings/hotgirlsraw.png new file mode 100644 index 000000000..5e607373e Binary files /dev/null and b/public/img/logos/filthykings/hotgirlsraw.png differ diff --git a/public/img/logos/filthykings/itsanal.png b/public/img/logos/filthykings/itsanal.png new file mode 100644 index 000000000..84102c2ff Binary files /dev/null and b/public/img/logos/filthykings/itsanal.png differ diff --git a/public/img/logos/filthykings/lazy/favicon.png b/public/img/logos/filthykings/lazy/favicon.png new file mode 100644 index 000000000..1b3c37bc2 Binary files /dev/null and b/public/img/logos/filthykings/lazy/favicon.png differ diff --git a/public/img/logos/filthykings/lazy/favicon_dark.png b/public/img/logos/filthykings/lazy/favicon_dark.png new file mode 100644 index 000000000..a4aee4f9b Binary files /dev/null and b/public/img/logos/filthykings/lazy/favicon_dark.png differ diff --git a/public/img/logos/filthykings/lazy/favicon_light.png b/public/img/logos/filthykings/lazy/favicon_light.png new file mode 100644 index 000000000..1b3c37bc2 Binary files /dev/null and b/public/img/logos/filthykings/lazy/favicon_light.png differ diff --git a/public/img/logos/filthykings/lazy/fillupmymom.png b/public/img/logos/filthykings/lazy/fillupmymom.png new file mode 100644 index 000000000..5e31e83b9 Binary files /dev/null and b/public/img/logos/filthykings/lazy/fillupmymom.png differ diff --git a/public/img/logos/filthykings/lazy/filthyblowjobs.png b/public/img/logos/filthykings/lazy/filthyblowjobs.png new file mode 100644 index 000000000..a7924655d Binary files /dev/null and b/public/img/logos/filthykings/lazy/filthyblowjobs.png differ diff --git a/public/img/logos/filthykings/lazy/filthykings.png b/public/img/logos/filthykings/lazy/filthykings.png old mode 100755 new mode 100644 index ffbd54caf..36f784233 Binary files a/public/img/logos/filthykings/lazy/filthykings.png and b/public/img/logos/filthykings/lazy/filthykings.png differ diff --git a/public/img/logos/filthykings/lazy/filthymassage.png b/public/img/logos/filthykings/lazy/filthymassage.png new file mode 100644 index 000000000..a3e86da20 Binary files /dev/null and b/public/img/logos/filthykings/lazy/filthymassage.png differ diff --git a/public/img/logos/filthykings/lazy/filthynewbies.png b/public/img/logos/filthykings/lazy/filthynewbies.png new file mode 100644 index 000000000..33ba1e2be Binary files /dev/null and b/public/img/logos/filthykings/lazy/filthynewbies.png differ diff --git a/public/img/logos/filthykings/lazy/filthypov.png b/public/img/logos/filthykings/lazy/filthypov.png new file mode 100644 index 000000000..22addc335 Binary files /dev/null and b/public/img/logos/filthykings/lazy/filthypov.png differ diff --git a/public/img/logos/filthykings/lazy/filthytaboo.png b/public/img/logos/filthykings/lazy/filthytaboo.png new file mode 100644 index 000000000..f9ad8e8a4 Binary files /dev/null and b/public/img/logos/filthykings/lazy/filthytaboo.png differ diff --git a/public/img/logos/filthykings/lazy/fkbts.png b/public/img/logos/filthykings/lazy/fkbts.png new file mode 100644 index 000000000..6baa5343b Binary files /dev/null and b/public/img/logos/filthykings/lazy/fkbts.png differ diff --git a/public/img/logos/filthykings/lazy/hotgirlsraw.png b/public/img/logos/filthykings/lazy/hotgirlsraw.png new file mode 100644 index 000000000..1c4003ae6 Binary files /dev/null and b/public/img/logos/filthykings/lazy/hotgirlsraw.png differ diff --git a/public/img/logos/filthykings/lazy/itsanal.png b/public/img/logos/filthykings/lazy/itsanal.png new file mode 100644 index 000000000..4296e61b4 Binary files /dev/null and b/public/img/logos/filthykings/lazy/itsanal.png differ diff --git a/public/img/logos/filthykings/lazy/network.png b/public/img/logos/filthykings/lazy/network.png old mode 100755 new mode 100644 index 594f70b26..36f784233 Binary files a/public/img/logos/filthykings/lazy/network.png and b/public/img/logos/filthykings/lazy/network.png differ diff --git a/public/img/logos/filthykings/misc/fill-up-my-mom_splash.png b/public/img/logos/filthykings/misc/fill-up-my-mom_splash.png new file mode 100644 index 000000000..46184ba7e Binary files /dev/null and b/public/img/logos/filthykings/misc/fill-up-my-mom_splash.png differ diff --git a/public/img/logos/filthykings/misc/filthy-blowjobs_cover.png b/public/img/logos/filthykings/misc/filthy-blowjobs_cover.png new file mode 100644 index 000000000..2e38693fb Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-blowjobs_cover.png differ diff --git a/public/img/logos/filthykings/misc/filthy-blowjobs_dark.png b/public/img/logos/filthykings/misc/filthy-blowjobs_dark.png new file mode 100644 index 000000000..4aa408c1f Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-blowjobs_dark.png differ diff --git a/public/img/logos/filthykings/misc/filthy-blowjobs_light.png b/public/img/logos/filthykings/misc/filthy-blowjobs_light.png new file mode 100644 index 000000000..c4aff61ee Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-blowjobs_light.png differ diff --git a/public/img/logos/filthykings/misc/filthy-massage_dark.png b/public/img/logos/filthykings/misc/filthy-massage_dark.png new file mode 100644 index 000000000..fbf853bfa Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-massage_dark.png differ diff --git a/public/img/logos/filthykings/misc/filthy-newbies_cover.png b/public/img/logos/filthykings/misc/filthy-newbies_cover.png new file mode 100644 index 000000000..0792ad8a7 Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-newbies_cover.png differ diff --git a/public/img/logos/filthykings/misc/filthy-newbies_dark.png b/public/img/logos/filthykings/misc/filthy-newbies_dark.png new file mode 100644 index 000000000..6e4c4197f Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-newbies_dark.png differ diff --git a/public/img/logos/filthykings/misc/filthy-newbies_light.png b/public/img/logos/filthykings/misc/filthy-newbies_light.png new file mode 100644 index 000000000..3885dc6b0 Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-newbies_light.png differ diff --git a/public/img/logos/filthykings/misc/filthy-pov_cropped.png b/public/img/logos/filthykings/misc/filthy-pov_cropped.png new file mode 100644 index 000000000..6363323d4 Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-pov_cropped.png differ diff --git a/public/img/logos/filthykings/misc/filthy-pov_uncropped.png b/public/img/logos/filthykings/misc/filthy-pov_uncropped.png new file mode 100644 index 000000000..b744fa527 Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-pov_uncropped.png differ diff --git a/public/img/logos/filthykings/misc/filthy-taboo_dark.png b/public/img/logos/filthykings/misc/filthy-taboo_dark.png new file mode 100644 index 000000000..a0a71b995 Binary files /dev/null and b/public/img/logos/filthykings/misc/filthy-taboo_dark.png differ diff --git a/public/img/logos/filthykings/misc/filthypov_light.png b/public/img/logos/filthykings/misc/filthypov_light.png new file mode 100644 index 000000000..0ae1aa86c Binary files /dev/null and b/public/img/logos/filthykings/misc/filthypov_light.png differ diff --git a/public/img/logos/filthykings/misc/hot-girls-raw_cover.png b/public/img/logos/filthykings/misc/hot-girls-raw_cover.png new file mode 100644 index 000000000..a35fce44f Binary files /dev/null and b/public/img/logos/filthykings/misc/hot-girls-raw_cover.png differ diff --git a/public/img/logos/filthykings/misc/its-anal_banner.jpg b/public/img/logos/filthykings/misc/its-anal_banner.jpg new file mode 100644 index 000000000..9c9e56fa2 Binary files /dev/null and b/public/img/logos/filthykings/misc/its-anal_banner.jpg differ diff --git a/public/img/logos/filthykings/thumbs/favicon.png b/public/img/logos/filthykings/thumbs/favicon.png new file mode 100644 index 000000000..1b3c37bc2 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/favicon.png differ diff --git a/public/img/logos/filthykings/thumbs/favicon_dark.png b/public/img/logos/filthykings/thumbs/favicon_dark.png new file mode 100644 index 000000000..a4aee4f9b Binary files /dev/null and b/public/img/logos/filthykings/thumbs/favicon_dark.png differ diff --git a/public/img/logos/filthykings/thumbs/favicon_light.png b/public/img/logos/filthykings/thumbs/favicon_light.png new file mode 100644 index 000000000..1b3c37bc2 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/favicon_light.png differ diff --git a/public/img/logos/filthykings/thumbs/fillupmymom.png b/public/img/logos/filthykings/thumbs/fillupmymom.png new file mode 100644 index 000000000..abdbc560f Binary files /dev/null and b/public/img/logos/filthykings/thumbs/fillupmymom.png differ diff --git a/public/img/logos/filthykings/thumbs/filthyblowjobs.png b/public/img/logos/filthykings/thumbs/filthyblowjobs.png new file mode 100644 index 000000000..fc1c2c80e Binary files /dev/null and b/public/img/logos/filthykings/thumbs/filthyblowjobs.png differ diff --git a/public/img/logos/filthykings/thumbs/filthykings.png b/public/img/logos/filthykings/thumbs/filthykings.png old mode 100755 new mode 100644 index 384dbf0e0..8f81581ae Binary files a/public/img/logos/filthykings/thumbs/filthykings.png and b/public/img/logos/filthykings/thumbs/filthykings.png differ diff --git a/public/img/logos/filthykings/thumbs/filthymassage.png b/public/img/logos/filthykings/thumbs/filthymassage.png new file mode 100644 index 000000000..c036f615b Binary files /dev/null and b/public/img/logos/filthykings/thumbs/filthymassage.png differ diff --git a/public/img/logos/filthykings/thumbs/filthynewbies.png b/public/img/logos/filthykings/thumbs/filthynewbies.png new file mode 100644 index 000000000..8dd0ced73 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/filthynewbies.png differ diff --git a/public/img/logos/filthykings/thumbs/filthypov.png b/public/img/logos/filthykings/thumbs/filthypov.png new file mode 100644 index 000000000..1665ef08b Binary files /dev/null and b/public/img/logos/filthykings/thumbs/filthypov.png differ diff --git a/public/img/logos/filthykings/thumbs/filthytaboo.png b/public/img/logos/filthykings/thumbs/filthytaboo.png new file mode 100644 index 000000000..0aa91b8e0 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/filthytaboo.png differ diff --git a/public/img/logos/filthykings/thumbs/fkbts.png b/public/img/logos/filthykings/thumbs/fkbts.png new file mode 100644 index 000000000..d36cbab97 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/fkbts.png differ diff --git a/public/img/logos/filthykings/thumbs/hotgirlsraw.png b/public/img/logos/filthykings/thumbs/hotgirlsraw.png new file mode 100644 index 000000000..8095febb3 Binary files /dev/null and b/public/img/logos/filthykings/thumbs/hotgirlsraw.png differ diff --git a/public/img/logos/filthykings/thumbs/itsanal.png b/public/img/logos/filthykings/thumbs/itsanal.png new file mode 100644 index 000000000..74343dcfe Binary files /dev/null and b/public/img/logos/filthykings/thumbs/itsanal.png differ diff --git a/public/img/logos/filthykings/thumbs/network.png b/public/img/logos/filthykings/thumbs/network.png old mode 100755 new mode 100644 index ae047922b..8f81581ae Binary files a/public/img/logos/filthykings/thumbs/network.png and b/public/img/logos/filthykings/thumbs/network.png differ diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 5b3a978b0..39881ff66 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -301,6 +301,19 @@ const networks = [ }, parent: 'mindgeek', }, + { + slug: 'filthykings', + name: 'Filthy Kings', + url: 'https://www.filthykings.com', + parent: 'gamma', + parameters: { + layout: 'api', + queryChannel: true, + scene: 'https://www.filthykings.com/en/video/filthykings', + referer: 'https://www.filthykings.com', + album: 'https://www.filthykings.com', + }, + }, { slug: 'firstanalquest', name: 'First Anal Quest', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 926f627de..b3987704a 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -3728,11 +3728,80 @@ const sites = [ slug: 'filthykings', name: 'Filthy Kings', url: 'https://www.filthykings.com', - parent: 'gamma', - independent: true, parameters: { - layout: 'api', + queryChannel: false, + filterNetwork: true, }, + parent: 'filthykings', + }, + { + slug: 'itsanal', + name: 'Its Anal', + url: 'https://www.itsanal.com', + tags: ['anal'], + parent: 'filthykings', + }, + { + slug: 'filthypov', + name: 'Filthy POV', + url: 'https://www.filthypov.com', + tags: ['pov'], + parameters: { + queryChannel: 'filthy-pov', + }, + parent: 'filthykings', + }, + { + slug: 'fkbts', + name: 'FK BTS', + url: 'https://www.filthykings.com', + tags: ['bts'], + parent: 'filthykings', + }, + { + slug: 'fillupmymom', + name: 'Fill Up My Mom', + url: 'https://www.fillupmymom.com', + tags: ['milf'], + parameters: { + queryChannel: 'fill-up-my-mom', + }, + parent: 'filthykings', + }, + { + slug: 'filthymassage', + name: 'Filthy Massage', + url: 'https://www.filthyMassage.com', + tags: ['massage'], + parent: 'filthykings', + }, + { + slug: 'filthytaboo', + name: 'Filthy Taboo', + url: 'https://www.filthytaboo.com', + parent: 'filthykings', + }, + { + slug: 'filthynewbies', + name: 'Filthy Newbies', + url: 'https://www.filthykings.com', + parent: 'filthykings', + }, + { + slug: 'filthyblowjobs', + name: 'Filthy Blowjobs', + url: 'https://www.filthykings.com', + tags: ['blowjob'], + parameters: { + queryChannel: 'filthyblowjob', + }, + parent: 'filthykings', + }, + { + slug: 'hotgirlsraw', + name: 'Hot Girls Raw', + url: 'https://www.hotgirlsraw.com', + parent: 'filthykings', }, // FIRST ANAL QUEST { diff --git a/src/deep.js b/src/deep.js index b52da4410..6eb2f7720 100755 --- a/src/deep.js +++ b/src/deep.js @@ -234,8 +234,6 @@ async function scrapeReleases(baseReleases, entitiesByHostname, type) { const parameters = getRecursiveParameters(entity); const preData = await entity.scraper.beforeFetchScenes(entity, parameters); - console.log('pre data', preData); - return [slug, { ...entity, preData }]; } diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index d609028fb..b7d5aa1f3 100755 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -143,8 +143,8 @@ async function getPhotos(albumPath, site, includeThumbnails = true) { } } -async function getFullPhotos(entryId, site) { - const res = await http.get(`${site.url}/media/signPhotoset/${entryId}`, { +async function getFullPhotos(entryId, site, parameters) { + const res = await http.get(`${parameters.album || site.url}/media/signPhotoset/${entryId}`, { headers: { 'X-Requested-With': 'XMLHttpRequest', }, @@ -193,8 +193,8 @@ async function getThumbs(entryId, site, parameters) { async function getPhotosApi(entryId, site, parameters) { const [photos, thumbs] = await Promise.all([ - getFullPhotos(entryId, site, parameters), - getThumbs(entryId, site, parameters), + getFullPhotos(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }), + getThumbs(entryId, site, parameters).catch(() => { logger.error(`Gamma scraper failed to fetch photos for ${entryId}`); return []; }), ]); return photos.concat(thumbs.slice(photos.length)); @@ -217,14 +217,14 @@ function curateTitle(title, channel) { return title.replace(new RegExp(`^\\s*${channel.name}\\s*[:|-]\\s`, 'i'), ''); } -async function scrapeApiReleases(json, site) { - return json.map((scene) => { - if (site.parameters?.extract && scene.sitename !== site.parameters.extract) { - return null; +async function scrapeApiReleases(json, site, options) { + return json.reduce((acc, scene) => { + if (options.parameters?.extract && scene.sitename !== options.parameters.extract) { + return acc; } - if (site.parameters?.filterExclusive && scene.availableOnSite.length > 1) { - return null; + if (options.parameters?.filterExclusive && scene.availableOnSite.length > 1) { + return acc; } const release = { @@ -238,8 +238,11 @@ async function scrapeApiReleases(json, site) { release.title = curateTitle(scene.title, site); release.path = `/${scene.url_title}/${release.entryId}`; - if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`; - else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`; + if (options.parameters?.scene) { + release.url = `${options.parameters.scene}${release.path}`; + } else if (site.url && options.parameters?.scene !== false) { + release.url = `${site.url}/en/video${release.path}`; + } release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate(); release.director = scene.directors[0]?.name || null; @@ -276,10 +279,24 @@ async function scrapeApiReleases(json, site) { ]; } + if (options.parameters.filterNetwork && scene.mainChannel) { + return { + ...acc, + unextracted: acc.unextracted.concat(release), + }; + } + + release.channel = slugify(scene.mainChannel?.id || scene.sitename, ''); // remove - // release.movie = `${site.url}/en/movie/${scene.url_movie_title}/${scene.movie_id}`; - return release; - }).filter(Boolean); + return { + ...acc, + scenes: acc.scenes.concat(release), + }; + }, { + scenes: [], + unextracted: [], + }); } function scrapeAll(scenes, site, networkUrl, hasTeaser = true) { @@ -476,7 +493,7 @@ async function scrapeReleaseApi(data, site, options, movieScenes) { release.scenes = await Promise.all(movieScenes.map((movieScene) => scrapeReleaseApi(movieScene, site, options))); } - release.channel = data.sitename; + release.channel = slugify(data.mainChannel?.id || data.sitename, ''); // remove - release.qualities = data.download_sizes; return release; @@ -638,11 +655,15 @@ async function fetchLatestApi(site, page = 1, options, preData, upcoming = false const referer = options.parameters?.referer || `${options.parameters?.networkReferer ? site.parent.url : site.url}/en/videos`; const { apiUrl } = await fetchApiCredentials(referer, site); + const params = `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]${options.parameters.queryChannel + ? `&filters=channels.id:${options.parameters.queryChannel === true ? site.slug : options.parameters.queryChannel}` + : `&filters=sitename:${site.slug}`}`; + const res = await http.post(apiUrl, { requests: [ { indexName: 'all_scenes', - params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug}`, // OR channels.id:${site.slug}`, + params, }, ], }, { @@ -654,7 +675,7 @@ async function fetchLatestApi(site, page = 1, options, preData, upcoming = false }); if (res.status === 200 && res.body.results?.[0]?.hits) { - return scrapeApiReleases(res.body.results[0].hits, site); + return scrapeApiReleases(res.body.results[0].hits, site, options); } return res.status; diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index b9c1e01fb..f7bc8f4e2 100755 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -2,12 +2,10 @@ const util = require('util'); const Promise = require('bluebird'); -const cheerio = require('cheerio'); -const moment = require('moment'); const unprint = require('unprint'); +const argv = require('../argv'); const qu = require('../utils/qu'); -const http = require('../utils/http'); const { heightToCm } = require('../utils/convert'); const slugify = require('../utils/slugify'); @@ -90,44 +88,23 @@ function scrapeAll(scenes, site, entryIdFromTitle) { }); } -function scrapeUpcoming(html, site) { - const $ = cheerio.load(html, { normalizeWhitespace: true }); - const scenesElements = $('#coming_soon_carousel').find('.table').toArray(); - - return scenesElements.map((element) => { +function scrapeUpcoming(scenes, channel) { + return scenes.map(({ query, html }) => { const release = {}; - release.entryId = $(element).find('.upcoming_updates_thumb').attr('id').match(/\d+/)[0]; + release.title = query.text('.overlay-text', { join: false })?.[0]; + release.date = query.date('.overlay-text', 'MM/DD/YYYY'); - const details = $(element).find('.update_details_comingsoon') - .eq(1) - .children() - .remove(); + release.actors = query.all('.update_models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); - release.title = details - .end() - .text() - .trim(); + release.poster = query.img('img') || query.img('img', { attribute: 'src0_1x' }); - release.actors = details - .text() - .trim() - .split(', '); + release.entryId = channel.parameters?.entryIdFromTitle ? slugify(release.title) : getEntryId(html); - release.date = moment - .utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY') - .toDate(); - - const photoElement = $(element).find('a img.thumbs'); - const posterPath = photoElement.attr('src'); - release.poster = posterPath.match(/^http/) ? posterPath : `${site.url}${posterPath}`; - - const videoClass = $(element).find('.update_thumbnail div').attr('class'); - const videoScript = $(element).find(`script:contains(${videoClass})`).html(); - - if (videoScript) { - release.teaser = videoScript.slice(videoScript.indexOf('https://'), videoScript.indexOf('.mp4') + 4); - } + // TODO: teaser return release; }); @@ -230,8 +207,11 @@ async function scrapeScene({ html, query }, context) { } // release.photos = async () => await getPhotos(release.entryId, context.entity); // probably no longer works on any site - // release.photos = query.imgs('#images img'); - release.photos = getPhotos(query, release, context); + if (argv.jjFullPhotos) { + release.photos = getPhotos(query, release, context); + } else { + release.photos = query.imgs('#images img'); + } if (query.exists('.update_dvds a')) { release.movie = { @@ -271,55 +251,6 @@ function scrapeMovie({ el, query }, url, site) { }; } -/* -function scrapeProfile(html, url, actorName, entity) { - const bio = document.querySelector('.model_bio').textContent; - const avatarEl = document.querySelector('.model_bio_pic img, .model_bio_thumb'); - - const profile = { - name: actorName, - }; - - const heightString = bio.match(/\d+ feet \d+ inches/); - const ageString = bio.match(/Age:\s*(\d{2})/); - const birthDateString = bio.match(/Age:\s*(\w+ \d{1,2}, \d{4})/); - const measurementsString = bio.match(/\w+-\d+-\d+/); - - if (birthDateString) profile.birthdate = qu.parseDate(birthDateString[1], 'MMMM D, YYYY'); - if (ageString) profile.age = Number(ageString[1]); - - if (heightString) profile.height = heightToCm(heightString[0]); - - if (measurementsString) { - const [bust, waist, hip] = measurementsString[0].split('-'); - - if (bust) profile.bust = bust; - if (waist) profile.waist = Number(waist); - if (hip) profile.hip = Number(hip); - } - - if (avatarEl) { - const avatarSources = [ - avatarEl.getAttribute('src0_3x'), - avatarEl.getAttribute('src0_2x'), - avatarEl.getAttribute('src0_1x'), - avatarEl.getAttribute('src0'), - avatarEl.getAttribute('src'), - ] - .filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images - .map((avatar) => qu.prefixUrl(avatar, entity.url)); - - if (avatarSources.length) profile.avatar = avatarSources; - } - - profile.releases = Array.from(document.querySelectorAll('.category_listing_block .update_details > a:first-child'), (el) => el.href); - - console.log(profile); - - return profile; -} -*/ - function scrapeProfile({ query }, url, name, entity) { const profile = { url }; @@ -368,13 +299,13 @@ async function fetchUpcoming(site) { if (site.parameters?.upcoming === false) return null; const url = site.parameters?.upcoming ? util.format(site.parameters.upcoming) : `${site.url}/trial/index.php`; - const res = await http.get(url); + const res = await unprint.get(url, { selectAll: '//img[contains(@alt, "Coming Soon")]/parent::div' }); - if (res.statusCode === 200) { - return scrapeUpcoming(res.body.toString(), site); + if (res.ok) { + return scrapeUpcoming(res.context, site); } - return res.statusCode; + return res.status; } async function fetchMovie(url, site) {