From f1a7f2f90539b3f0ac65cf2cc395c19a78daacb7 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Wed, 2 Feb 2022 23:51:15 +0100 Subject: [PATCH] Added Gamma movie API scraper. --- .../zerotolerance/lazy/addicted2girls.png | Bin 2242 -> 2242 bytes .../img/logos/zerotolerance/lazy/favicon.png | Bin 7143 -> 7143 bytes .../logos/zerotolerance/lazy/favicon_dark.png | Bin 7143 -> 7143 bytes .../zerotolerance/lazy/favicon_light.png | Bin 7143 -> 7143 bytes .../img/logos/zerotolerance/lazy/genderx.png | Bin 2012 -> 2012 bytes .../img/logos/zerotolerance/lazy/network.png | Bin 3915 -> 3915 bytes .../zerotolerance/lazy/zerotolerance.png | Bin 2191 -> 2191 bytes .../zerotolerance/lazy/zerotolerancefilms.png | Bin 2157 -> 2157 bytes .../zerotolerance/thumbs/addicted2girls.png | Bin 9740 -> 9740 bytes .../logos/zerotolerance/thumbs/favicon.png | Bin 7143 -> 7143 bytes .../zerotolerance/thumbs/favicon_dark.png | Bin 7143 -> 7143 bytes .../zerotolerance/thumbs/favicon_light.png | Bin 7143 -> 7143 bytes .../logos/zerotolerance/thumbs/genderx.png | Bin 4905 -> 4905 bytes .../logos/zerotolerance/thumbs/network.png | Bin 37169 -> 37169 bytes .../zerotolerance/thumbs/zerotolerance.png | Bin 12373 -> 12373 bytes .../thumbs/zerotolerancefilms.png | Bin 16144 -> 16144 bytes seeds/02_sites.js | 12 ++- src/scrapers/gamma.js | 99 +++++++++++------- 18 files changed, 71 insertions(+), 40 deletions(-) diff --git a/public/img/logos/zerotolerance/lazy/addicted2girls.png b/public/img/logos/zerotolerance/lazy/addicted2girls.png index f8c3c98b6befdb3f2b20a49c81a015535dbcb71e..14f3be66e57ccd3bdce4d6bc90826589f19ab727 100644 GIT binary patch delta 20 bcmX>kct~)=E*5q%2IK1La)!+ZSr}OWO2h_F delta 20 ccmX>kct~)=E*5rCF>c@GM+7z>WMO0h08Cc~mjD0& diff --git a/public/img/logos/zerotolerance/lazy/favicon.png b/public/img/logos/zerotolerance/lazy/favicon.png index 293ddec1f3e28bfd15170a5c3ab8ee7572341c49..dcf6c2323859d5a34871c1f796d3e9c33803af46 100644 GIT binary patch delta 20 bcmaEE{@i?nB@4S4gK>3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeA3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeA3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeAV!Z delta 20 ccmcb^e}{j=M`m_WF>c@GM+7$iV1B|309CXI6#xJL diff --git a/public/img/logos/zerotolerance/lazy/network.png b/public/img/logos/zerotolerance/lazy/network.png index 8fd3644c9ed5cc82ecdfdf0e7e8af7e59b923208..8adebbce15bfc44218fe7e810360669cf079016a 100644 GIT binary patch delta 20 bcmX>tcUo=(2NSy(gK>3rIm2dtCL3M=K@|m8 delta 20 bcmX>tcUo=(2NS!f7`N~8BLbWGnQV9gLZ}6w diff --git a/public/img/logos/zerotolerance/lazy/zerotolerance.png b/public/img/logos/zerotolerance/lazy/zerotolerance.png index c0a1d8ec926490337f8e896e36eb703dd2a631d1..59ca977b22da2b851488871518eb20cfd55ee35a 100644 GIT binary patch delta 20 bcmeAd>=)cn#lkMeU|d~Y&ak=)cn#lkKs#_hZOh`{DX7CsgLKpzFZ diff --git a/public/img/logos/zerotolerance/lazy/zerotolerancefilms.png b/public/img/logos/zerotolerance/lazy/zerotolerancefilms.png index 5e07f731734c714b27c054e4cc2d663e9eac379d..1b99b1f641620def3475b6fe46df1bc32e514e19 100644 GIT binary patch delta 20 bcmaDW@K#_$Ckwk6gK>3rIm6}&EO(g!PZkFp delta 20 ccmaDW@K#_$Ckwl%7`N~8BLbTzu-s(^08pt1WB>pF diff --git a/public/img/logos/zerotolerance/thumbs/addicted2girls.png b/public/img/logos/zerotolerance/thumbs/addicted2girls.png index cbcb27b77ab6efc167dbecb2a95914636e020730..180441718f3a2a2051e5ea51745332546ef440c2 100644 GIT binary patch delta 20 bcmeD2>G9dX!Ne}cU|d~Y&aj!E>8~;XKg$LT delta 20 bcmeD2>G9dX!Ne{q#_hZOh`?rkroYMnL0$$_ diff --git a/public/img/logos/zerotolerance/thumbs/favicon.png b/public/img/logos/zerotolerance/thumbs/favicon.png index 293ddec1f3e28bfd15170a5c3ab8ee7572341c49..dcf6c2323859d5a34871c1f796d3e9c33803af46 100644 GIT binary patch delta 20 bcmaEE{@i?nB@4S4gK>3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeA3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeA3rIm2cLmgV9AO=bq1 delta 20 ccmaEE{@i?nB@4T#7`N~8BLbTpSeA3rIm2ciHd$T(JoW_! delta 20 bcmZ3fwo+{aD;vA07`N~8BLbUw*kpMDK8XcR diff --git a/public/img/logos/zerotolerance/thumbs/network.png b/public/img/logos/zerotolerance/thumbs/network.png index 93391395805dcb3649b06fb00d41ac428dd7b33a..137e9a8eb360ab0a78193a8a213cc3c1385b805f 100644 GIT binary patch delta 22 dcmdnEh-u>@rVSiS>|zYY)z#$;oB5fP`T@rVSiS?4n}azRQmYZ02WD>IVQ-?*`uh diff --git a/public/img/logos/zerotolerance/thumbs/zerotolerance.png b/public/img/logos/zerotolerance/thumbs/zerotolerance.png index 48e7d347faf016e93630a754d9d887c4cda3bbc3..9916977e5678c1c5ad49509d88f10db6f4b4dc28 100644 GIT binary patch delta 20 bcmcbba5Z5A2NSy(gK>3rIm2dtCO16*Ob!Mb delta 20 bcmcbba5Z5A2NS!f7`N~8BLbWGncVaMO`!&2 diff --git a/public/img/logos/zerotolerance/thumbs/zerotolerancefilms.png b/public/img/logos/zerotolerance/thumbs/zerotolerancefilms.png index 3ac681b0d52b7108d65c4cdc24fa67bb1b1f4fc4..56f067a5d88fbdc0269c4f4ab65d202b7a57a2ec 100644 GIT binary patch delta 20 bcmbPGH=%9=2NSy(gK>3rIm2dtCPrHTMU4fP delta 20 bcmbPGH=%9=2NS!f7`N~8BLbWGnHX&WM<4~> diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 7eb2baa5..5d36d9c8 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -10922,16 +10922,17 @@ const sites = [ parent: 'zerotolerance', parameters: { scene: 'https://www.addicted2girls.com/en/video/addicted2girls', + movie: 'https://www.addicted2girls.com/en/dvd', }, }, { - slug: 'genderx', - name: 'GenderX', - url: 'https://www.genderx.com', + slug: 'genderxfilms', + name: 'GenderXFilms', + url: 'https://www.genderxfilms.com', tags: ['transsexual'], parent: 'zerotolerance', parameters: { - scene: 'https://www.genderx.com/en/video', + movie: 'https://www.genderxfilms.com/en/dvd', }, }, { @@ -10939,6 +10940,9 @@ const sites = [ name: 'Zero Tolerance Films', url: 'https://www.zerotolerancefilms.com', parent: 'zerotolerance', + parameters: { + scene: 'https://www.zerotolerancefilms.com/en/video/zerotolerancefilms', + }, }, ]; diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 4e5ee037..0ecdbd8d 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -200,6 +200,18 @@ async function getPhotosApi(entryId, site, parameters) { return photos.concat(thumbs.slice(photos.length)); } +function getImageSources(source) { + return [ + `https://images-fame.gammacdn.com/movies${source}`, + `https://images01-fame.gammacdn.com/movies${source}`, + `https://images02-fame.gammacdn.com/movies${source}`, + `https://images03-fame.gammacdn.com/movies${source}`, + `https://images04-fame.gammacdn.com/movies${source}`, + `https://images-evilangel.gammacdn.com/movies${source}`, + `https://transform.gammacdn.com/movies${source}`, + ]; +} + async function scrapeApiReleases(json, site) { return json.map((scene) => { if (site.parameters?.extract && scene.sitename !== site.parameters.extract) { @@ -402,13 +414,14 @@ async function scrapeScene({ query }, url, channel, baseRelease, mobileItem, opt return release; } -async function scrapeSceneApi(data, site, options) { +async function scrapeReleaseApi(data, site, options) { const release = {}; - release.entryId = data.clip_id; + release.entryId = data.clip_id || data.movie_id; release.title = data.title; release.duration = data.length; - release.date = new Date(data.date * 1000) || qu.parseDate(data.release_date, 'YYYY-MM-DD'); + release.date = (data.date && new Date(data.date * 1000)) || qu.parseDate(data.release_date || data.last_modified, 'YYYY-MM-DD'); + release.director = data.directors[0]?.name || null; release.actors = data.actors.map((actor) => ({ entryId: actor.actor_id, @@ -423,33 +436,9 @@ async function scrapeSceneApi(data, site, options) { if (data.pictures) { release.poster = [ - ...(data.pictures['1920x1080'] ? [ - `https://images-fame.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://images01-fame.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://images02-fame.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://images03-fame.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://images04-fame.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://images-evilangel.gammacdn.com/movies${data.pictures['1920x1080']}`, - `https://transform.gammacdn.com/movies${data.pictures['1920x1080']}`, - ] : []), - ...(data.pictures.resized ? [ - `https://images-fame.gammacdn.com/movies${data.pictures.resized}`, - `https://images01-fame.gammacdn.com/movies${data.pictures.resized}`, - `https://images02-fame.gammacdn.com/movies${data.pictures.resized}`, - `https://images03-fame.gammacdn.com/movies${data.pictures.resized}`, - `https://images04-fame.gammacdn.com/movies${data.pictures.resized}`, - `https://images-evilangel.gammacdn.com/movies${data.pictures.resized}`, - `https://transform.gammacdn.com/movies${data.pictures.resized}`, - ] : []), - ...(data.pictures['960x544'] ? [ - `https://images-fame.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://images01-fame.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://images02-fame.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://images03-fame.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://images04-fame.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://images-evilangel.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - `https://transform.gammacdn.com/movies${data.pictures['960x544']}?width=1950`, - ] : []), + ...(data.pictures['1920x1080'] ? getImageSources(data.pictures['1920x1080']) : []), + ...(data.pictures.resized ? getImageSources(data.pictures.resized) : []), + ...(data.pictures['960x544'] ? getImageSources(data.pictures['960x544']) : []), ]; } @@ -457,15 +446,22 @@ async function scrapeSceneApi(data, site, options) { release.photos = await getPhotosApi(data.photoset_id, site, options.parameters); } + if (data.cover_path) { + release.covers = [ + getImageSources(`/${data.cover_path}_front_400x625.jpg?width=450&height=636&format=webp`), + getImageSources(`/${data.cover_path}_back_400x625.jpg?width=450&height=636&format=webp`), + ]; + } + if (data.trailers) { release.trailer = Object.entries(data.trailers).map(([quality, source]) => ({ src: source, quality })); } - if (data.movie_id) { + if (data.movie_id && !data.movie_path) { release.movie = { entryId: data.movie_id, title: data.movie_title, - url: qu.prefixUrl(`/en/movie/${data.url_movie_title}/${data.movie_id}`, site.url), + url: qu.prefixUrl(`${data.url_movie_title}/${data.movie_id}`, options.parameters.movie ? options.parameters.movie : `${site.url}/en/movie`), }; } @@ -642,8 +638,6 @@ async function fetchLatestApi(site, page = 1, preData, include, upcoming = false encodeJSON: true, }); - console.log(res.body.results); - if (res.status === 200 && res.body.results?.[0]?.hits) { return scrapeApiReleases(res.body.results[0].hits, site); } @@ -681,7 +675,39 @@ async function fetchSceneApi(url, site, baseRelease, options) { }); if (res.status === 200 && res.body.results?.[0]?.hits) { - return scrapeSceneApi(res.body.results[0].hits[0], site, options); + return scrapeReleaseApi(res.body.results[0].hits[0], site, options); + } + + return res.status; +} + +async function fetchMovieApi(url, site, baseRelease, options) { + const referer = options.parameters?.referer || `${site.parameters?.networkReferer ? site.parent.url : site.url}/en/movies`; + const { apiUrl } = await fetchApiCredentials(referer, site); + + const entryId = (baseRelease?.path || new URL(url).pathname).match(/\/(\d{2,})(\/|$)/)?.[1]; + + const res = await http.post(apiUrl, { + requests: [ + { + indexName: 'all_movies', + params: `query=&page=0&facets=[]&tagFilters=&facetFilters=[["movie_id:${entryId}"]]`, + }, + { + indexName: 'all_movies', + params: 'query=&page=0&hitsPerPage=1&attributesToRetrieve=[]&attributesToHighlight=[]&attributesToSnippet=[]&tagFilters=&analytics=false&clickAnalytics=false&facets=clip_id', + }, + ], + }, { + headers: { + Referer: referer, + }, + }, { + encodeJSON: true, + }); + + if (res.status === 200 && res.body.results?.[0]?.hits) { + return scrapeReleaseApi(res.body.results[0].hits[0], site, options); } return res.status; @@ -894,7 +920,8 @@ module.exports = { fetchProfile: fetchApiProfile, // fetchScene, fetchScene: fetchSceneApi, - scrapeMovie, + // scrapeMovie, + fetchMovie: fetchMovieApi, }, getPhotos, scrapeApiProfile,