diff --git a/package-lock.json b/package-lock.json index a40e5497..965c3679 100755 --- a/package-lock.json +++ b/package-lock.json @@ -78,7 +78,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.8.2", + "unprint": "^0.9.1", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", @@ -17004,9 +17004,9 @@ } }, "node_modules/unprint": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.8.2.tgz", - "integrity": "sha512-mCKPDPwtuECbXJJLQbDn2FFbydr6fLKytyS3pymbxcTh2dkk7NFypMjR7qjU2Uv9Fl91hSE48SjYMsWHNKpp4w==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.9.1.tgz", + "integrity": "sha512-TEdPfly6qOST3Vessa9gNFFkJsSsdWe04k0FCU5XuvdpCSb8eVRrAvLouuyJI/GkIhrn6ZHc9VgX/gRW/R5UcQ==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", @@ -31449,9 +31449,9 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "unprint": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.8.2.tgz", - "integrity": "sha512-mCKPDPwtuECbXJJLQbDn2FFbydr6fLKytyS3pymbxcTh2dkk7NFypMjR7qjU2Uv9Fl91hSE48SjYMsWHNKpp4w==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.9.1.tgz", + "integrity": "sha512-TEdPfly6qOST3Vessa9gNFFkJsSsdWe04k0FCU5XuvdpCSb8eVRrAvLouuyJI/GkIhrn6ZHc9VgX/gRW/R5UcQ==", "requires": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index cc0e78df..3f4a0174 100755 --- a/package.json +++ b/package.json @@ -137,7 +137,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.8.2", + "unprint": "^0.9.1", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", diff --git a/public/img/logos/newsensations/lazy/ashlynnbrooke.png b/public/img/logos/newsensations/lazy/ashlynnbrooke.png old mode 100755 new mode 100644 index eaffc97c..a253246b Binary files a/public/img/logos/newsensations/lazy/ashlynnbrooke.png and b/public/img/logos/newsensations/lazy/ashlynnbrooke.png differ diff --git a/public/img/logos/newsensations/lazy/favicon.png b/public/img/logos/newsensations/lazy/favicon.png old mode 100755 new mode 100644 index aeb21d53..025a0c62 Binary files a/public/img/logos/newsensations/lazy/favicon.png and b/public/img/logos/newsensations/lazy/favicon.png differ diff --git a/public/img/logos/newsensations/lazy/favicon_dark.png b/public/img/logos/newsensations/lazy/favicon_dark.png new file mode 100644 index 00000000..993b9ddc Binary files /dev/null and b/public/img/logos/newsensations/lazy/favicon_dark.png differ diff --git a/public/img/logos/newsensations/lazy/favicon_light.png b/public/img/logos/newsensations/lazy/favicon_light.png new file mode 100644 index 00000000..025a0c62 Binary files /dev/null and b/public/img/logos/newsensations/lazy/favicon_light.png differ diff --git a/public/img/logos/newsensations/lazy/fourfingerclub.png b/public/img/logos/newsensations/lazy/fourfingerclub.png old mode 100755 new mode 100644 index cda93db6..7b8076f2 Binary files a/public/img/logos/newsensations/lazy/fourfingerclub.png and b/public/img/logos/newsensations/lazy/fourfingerclub.png differ diff --git a/public/img/logos/newsensations/lazy/freshouttahighschool.png b/public/img/logos/newsensations/lazy/freshouttahighschool.png old mode 100755 new mode 100644 index fcaffc24..51661fc4 Binary files a/public/img/logos/newsensations/lazy/freshouttahighschool.png and b/public/img/logos/newsensations/lazy/freshouttahighschool.png differ diff --git a/public/img/logos/newsensations/lazy/heavyhandfuls.png b/public/img/logos/newsensations/lazy/heavyhandfuls.png old mode 100755 new mode 100644 index 094d1974..b29d1059 Binary files a/public/img/logos/newsensations/lazy/heavyhandfuls.png and b/public/img/logos/newsensations/lazy/heavyhandfuls.png differ diff --git a/public/img/logos/newsensations/lazy/hotwifexxx.png b/public/img/logos/newsensations/lazy/hotwifexxx.png old mode 100755 new mode 100644 index d23e1766..e81e6f63 Binary files a/public/img/logos/newsensations/lazy/hotwifexxx.png and b/public/img/logos/newsensations/lazy/hotwifexxx.png differ diff --git a/public/img/logos/newsensations/lazy/jizzbomb.png b/public/img/logos/newsensations/lazy/jizzbomb.png old mode 100755 new mode 100644 index 470e8577..314df261 Binary files a/public/img/logos/newsensations/lazy/jizzbomb.png and b/public/img/logos/newsensations/lazy/jizzbomb.png differ diff --git a/public/img/logos/newsensations/lazy/logo.png b/public/img/logos/newsensations/lazy/logo.png old mode 100755 new mode 100644 index 82bc8953..a889f86e Binary files a/public/img/logos/newsensations/lazy/logo.png and b/public/img/logos/newsensations/lazy/logo.png differ diff --git a/public/img/logos/newsensations/lazy/network.png b/public/img/logos/newsensations/lazy/network.png old mode 100755 new mode 100644 index e15520d8..eee22bf2 Binary files a/public/img/logos/newsensations/lazy/network.png and b/public/img/logos/newsensations/lazy/network.png differ diff --git a/public/img/logos/newsensations/lazy/nsfamilyxxx.png b/public/img/logos/newsensations/lazy/nsfamilyxxx.png old mode 100755 new mode 100644 index 9c3d8087..cd485f65 Binary files a/public/img/logos/newsensations/lazy/nsfamilyxxx.png and b/public/img/logos/newsensations/lazy/nsfamilyxxx.png differ diff --git a/public/img/logos/newsensations/lazy/parodypass.png b/public/img/logos/newsensations/lazy/parodypass.png old mode 100755 new mode 100644 index 646b13a0..316e5ba3 Binary files a/public/img/logos/newsensations/lazy/parodypass.png and b/public/img/logos/newsensations/lazy/parodypass.png differ diff --git a/public/img/logos/newsensations/lazy/shanedieselsbangingbabes.png b/public/img/logos/newsensations/lazy/shanedieselsbangingbabes.png old mode 100755 new mode 100644 index fecd403c..3ad97b1b Binary files a/public/img/logos/newsensations/lazy/shanedieselsbangingbabes.png and b/public/img/logos/newsensations/lazy/shanedieselsbangingbabes.png differ diff --git a/public/img/logos/newsensations/lazy/shanedieselxxx.png b/public/img/logos/newsensations/lazy/shanedieselxxx.png new file mode 100644 index 00000000..1575f9d1 Binary files /dev/null and b/public/img/logos/newsensations/lazy/shanedieselxxx.png differ diff --git a/public/img/logos/newsensations/lazy/stretchedoutsnatch.png b/public/img/logos/newsensations/lazy/stretchedoutsnatch.png old mode 100755 new mode 100644 index 76237a9c..16e67f04 Binary files a/public/img/logos/newsensations/lazy/stretchedoutsnatch.png and b/public/img/logos/newsensations/lazy/stretchedoutsnatch.png differ diff --git a/public/img/logos/newsensations/lazy/tabutales.png b/public/img/logos/newsensations/lazy/tabutales.png old mode 100755 new mode 100644 index da124180..1bb775c5 Binary files a/public/img/logos/newsensations/lazy/tabutales.png and b/public/img/logos/newsensations/lazy/tabutales.png differ diff --git a/public/img/logos/newsensations/lazy/talesfromtheedge.png b/public/img/logos/newsensations/lazy/talesfromtheedge.png old mode 100755 new mode 100644 index 50d82dea..a866d498 Binary files a/public/img/logos/newsensations/lazy/talesfromtheedge.png and b/public/img/logos/newsensations/lazy/talesfromtheedge.png differ diff --git a/public/img/logos/newsensations/lazy/thelesbianexperience.png b/public/img/logos/newsensations/lazy/thelesbianexperience.png old mode 100755 new mode 100644 index c70f4f21..5369ad05 Binary files a/public/img/logos/newsensations/lazy/thelesbianexperience.png and b/public/img/logos/newsensations/lazy/thelesbianexperience.png differ diff --git a/public/img/logos/newsensations/lazy/theromanceseries.png b/public/img/logos/newsensations/lazy/theromanceseries.png old mode 100755 new mode 100644 index 82bc8953..a889f86e Binary files a/public/img/logos/newsensations/lazy/theromanceseries.png and b/public/img/logos/newsensations/lazy/theromanceseries.png differ diff --git a/public/img/logos/newsensations/lazy/unlimitedmilfs.png b/public/img/logos/newsensations/lazy/unlimitedmilfs.png old mode 100755 new mode 100644 index 771783b7..65d9f991 Binary files a/public/img/logos/newsensations/lazy/unlimitedmilfs.png and b/public/img/logos/newsensations/lazy/unlimitedmilfs.png differ diff --git a/public/img/logos/newsensations/shanedieselxxx.png b/public/img/logos/newsensations/shanedieselxxx.png new file mode 100644 index 00000000..173e3da5 Binary files /dev/null and b/public/img/logos/newsensations/shanedieselxxx.png differ diff --git a/public/img/logos/newsensations/thumbs/ashlynnbrooke.png b/public/img/logos/newsensations/thumbs/ashlynnbrooke.png old mode 100755 new mode 100644 index a7bb20de..41a34fe8 Binary files a/public/img/logos/newsensations/thumbs/ashlynnbrooke.png and b/public/img/logos/newsensations/thumbs/ashlynnbrooke.png differ diff --git a/public/img/logos/newsensations/thumbs/favicon.png b/public/img/logos/newsensations/thumbs/favicon.png old mode 100755 new mode 100644 index aeb21d53..025a0c62 Binary files a/public/img/logos/newsensations/thumbs/favicon.png and b/public/img/logos/newsensations/thumbs/favicon.png differ diff --git a/public/img/logos/newsensations/thumbs/favicon_dark.png b/public/img/logos/newsensations/thumbs/favicon_dark.png new file mode 100644 index 00000000..993b9ddc Binary files /dev/null and b/public/img/logos/newsensations/thumbs/favicon_dark.png differ diff --git a/public/img/logos/newsensations/thumbs/favicon_light.png b/public/img/logos/newsensations/thumbs/favicon_light.png new file mode 100644 index 00000000..025a0c62 Binary files /dev/null and b/public/img/logos/newsensations/thumbs/favicon_light.png differ diff --git a/public/img/logos/newsensations/thumbs/fourfingerclub.png b/public/img/logos/newsensations/thumbs/fourfingerclub.png old mode 100755 new mode 100644 index 33ee58b9..0ef81690 Binary files a/public/img/logos/newsensations/thumbs/fourfingerclub.png and b/public/img/logos/newsensations/thumbs/fourfingerclub.png differ diff --git a/public/img/logos/newsensations/thumbs/freshouttahighschool.png b/public/img/logos/newsensations/thumbs/freshouttahighschool.png old mode 100755 new mode 100644 index 07c922d5..97a72a16 Binary files a/public/img/logos/newsensations/thumbs/freshouttahighschool.png and b/public/img/logos/newsensations/thumbs/freshouttahighschool.png differ diff --git a/public/img/logos/newsensations/thumbs/heavyhandfuls.png b/public/img/logos/newsensations/thumbs/heavyhandfuls.png old mode 100755 new mode 100644 index fccccbf5..f4675f8a Binary files a/public/img/logos/newsensations/thumbs/heavyhandfuls.png and b/public/img/logos/newsensations/thumbs/heavyhandfuls.png differ diff --git a/public/img/logos/newsensations/thumbs/hotwifexxx.png b/public/img/logos/newsensations/thumbs/hotwifexxx.png old mode 100755 new mode 100644 index 7b1b96c0..df4ce195 Binary files a/public/img/logos/newsensations/thumbs/hotwifexxx.png and b/public/img/logos/newsensations/thumbs/hotwifexxx.png differ diff --git a/public/img/logos/newsensations/thumbs/jizzbomb.png b/public/img/logos/newsensations/thumbs/jizzbomb.png old mode 100755 new mode 100644 index 7f13bfa1..dabd723b Binary files a/public/img/logos/newsensations/thumbs/jizzbomb.png and b/public/img/logos/newsensations/thumbs/jizzbomb.png differ diff --git a/public/img/logos/newsensations/thumbs/logo.png b/public/img/logos/newsensations/thumbs/logo.png old mode 100755 new mode 100644 index 536f76bf..08be4163 Binary files a/public/img/logos/newsensations/thumbs/logo.png and b/public/img/logos/newsensations/thumbs/logo.png differ diff --git a/public/img/logos/newsensations/thumbs/network.png b/public/img/logos/newsensations/thumbs/network.png old mode 100755 new mode 100644 index caa348f2..3ef6c1e6 Binary files a/public/img/logos/newsensations/thumbs/network.png and b/public/img/logos/newsensations/thumbs/network.png differ diff --git a/public/img/logos/newsensations/thumbs/nsfamilyxxx.png b/public/img/logos/newsensations/thumbs/nsfamilyxxx.png old mode 100755 new mode 100644 index d8b4e50e..cdd494c3 Binary files a/public/img/logos/newsensations/thumbs/nsfamilyxxx.png and b/public/img/logos/newsensations/thumbs/nsfamilyxxx.png differ diff --git a/public/img/logos/newsensations/thumbs/parodypass.png b/public/img/logos/newsensations/thumbs/parodypass.png old mode 100755 new mode 100644 index 2aa7533f..28fcc4c5 Binary files a/public/img/logos/newsensations/thumbs/parodypass.png and b/public/img/logos/newsensations/thumbs/parodypass.png differ diff --git a/public/img/logos/newsensations/thumbs/shanedieselsbangingbabes.png b/public/img/logos/newsensations/thumbs/shanedieselsbangingbabes.png old mode 100755 new mode 100644 index ff47beeb..2ed8cd4b Binary files a/public/img/logos/newsensations/thumbs/shanedieselsbangingbabes.png and b/public/img/logos/newsensations/thumbs/shanedieselsbangingbabes.png differ diff --git a/public/img/logos/newsensations/thumbs/shanedieselxxx.png b/public/img/logos/newsensations/thumbs/shanedieselxxx.png new file mode 100644 index 00000000..834c29fb Binary files /dev/null and b/public/img/logos/newsensations/thumbs/shanedieselxxx.png differ diff --git a/public/img/logos/newsensations/thumbs/stretchedoutsnatch.png b/public/img/logos/newsensations/thumbs/stretchedoutsnatch.png old mode 100755 new mode 100644 index 3ff9f5df..96611150 Binary files a/public/img/logos/newsensations/thumbs/stretchedoutsnatch.png and b/public/img/logos/newsensations/thumbs/stretchedoutsnatch.png differ diff --git a/public/img/logos/newsensations/thumbs/tabutales.png b/public/img/logos/newsensations/thumbs/tabutales.png old mode 100755 new mode 100644 index c6be7689..2580845d Binary files a/public/img/logos/newsensations/thumbs/tabutales.png and b/public/img/logos/newsensations/thumbs/tabutales.png differ diff --git a/public/img/logos/newsensations/thumbs/talesfromtheedge.png b/public/img/logos/newsensations/thumbs/talesfromtheedge.png old mode 100755 new mode 100644 index aedb7d3e..1e9e1d46 Binary files a/public/img/logos/newsensations/thumbs/talesfromtheedge.png and b/public/img/logos/newsensations/thumbs/talesfromtheedge.png differ diff --git a/public/img/logos/newsensations/thumbs/thelesbianexperience.png b/public/img/logos/newsensations/thumbs/thelesbianexperience.png old mode 100755 new mode 100644 index 98fc72d0..d22fc9f3 Binary files a/public/img/logos/newsensations/thumbs/thelesbianexperience.png and b/public/img/logos/newsensations/thumbs/thelesbianexperience.png differ diff --git a/public/img/logos/newsensations/thumbs/theromanceseries.png b/public/img/logos/newsensations/thumbs/theromanceseries.png old mode 100755 new mode 100644 index 536f76bf..08be4163 Binary files a/public/img/logos/newsensations/thumbs/theromanceseries.png and b/public/img/logos/newsensations/thumbs/theromanceseries.png differ diff --git a/public/img/logos/newsensations/thumbs/unlimitedmilfs.png b/public/img/logos/newsensations/thumbs/unlimitedmilfs.png old mode 100755 new mode 100644 index bd5af522..d05d37c2 Binary files a/public/img/logos/newsensations/thumbs/unlimitedmilfs.png and b/public/img/logos/newsensations/thumbs/unlimitedmilfs.png differ diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 48832940..ae5b4462 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -6856,16 +6856,9 @@ const sites = [ parent: 'newsensations', parameters: { siteId: 'hwxxx', - block: true, + layout: 'block', }, }, - { - slug: 'tabutales', - name: 'Tabu Tales', - url: 'https://www.thetabutales.com', - parent: 'newsensations', - parameters: { siteId: 'tt' }, - }, { slug: 'nsfamilyxxx', name: 'Family XXX', @@ -6874,7 +6867,7 @@ const sites = [ tags: ['family'], parameters: { siteId: 'famxxx', - block: true, + layout: 'block', }, }, { @@ -6883,7 +6876,10 @@ const sites = [ url: 'https://www.thelesbianexperience.com', parent: 'newsensations', tags: ['lesbian'], - parameters: { siteId: 'tle' }, + parameters: { + siteId: 'tle', + layout: 'block', + }, }, { slug: 'theromanceseries', @@ -6892,13 +6888,6 @@ const sites = [ parent: 'newsensations', parameters: { siteId: 'rs' }, }, - { - slug: 'talesfromtheedge', - name: 'Tales From The Edge', - url: 'thetalesfromtheedge', - parent: 'newsensations', - parameters: { siteId: 'ttfte' }, - }, { slug: 'parodypass', name: 'Parody Pass', @@ -6907,11 +6896,15 @@ const sites = [ parameters: { siteId: 'pp' }, }, { - slug: 'shanedieselsbangingbabes', - name: 'Shane Diesel\'s Banging Babes', - url: 'http://shanedieselsbangingbabes.com', + slug: 'shanedieselxxx', + name: 'Shane Diesel XXX', + alias: ['shane diesel\'s banging babes', 'sdbb'], + url: 'https://shanedieselxxx.com', parent: 'newsensations', - parameters: { siteId: 'sdbb' }, + parameters: { + siteId: 'sdxxx', + layout: 'block', + }, }, { slug: 'unlimitedmilfs', @@ -6961,7 +6954,25 @@ const sites = [ name: 'Fresh Outta High School', url: 'https://www.freshouttahighschool.com', parent: 'newsensations', - parameters: { siteId: 'fohs' }, + parameters: { + siteId: 'fohs', + layout: 'block', + }, + }, + // merged with Family XXX or main site + { + slug: 'tabutales', + name: 'Tabu Tales', + url: 'https://www.thetabutales.com', + parent: 'newsensations', + parameters: { siteId: 'tt' }, + }, + { + slug: 'talesfromtheedge', + name: 'Tales From The Edge', + url: 'https://www.thetalesfromtheedge.com', + parent: 'newsensations', + parameters: { siteId: 'ttfte' }, }, // NUBILES { diff --git a/src/deep.js b/src/deep.js index bbef8dd5..83029eaf 100755 --- a/src/deep.js +++ b/src/deep.js @@ -113,9 +113,6 @@ function fetchMovie(scraper, url, entity, baseRelease, options) { async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') { const entity = baseRelease.entity || entitiesByHostname[urlToHostname(baseRelease.url)]; - console.log(entitiesByHostname); - console.log(entity); - if (!entity) { logger.warn(`No entity available for ${baseRelease.url}`); return baseRelease; diff --git a/src/entities.js b/src/entities.js index 86700767..e1d252c7 100755 --- a/src/entities.js +++ b/src/entities.js @@ -103,7 +103,7 @@ function urlToHostname(url) { try { const hostname = new URL(url) .hostname - .match(/(www\.)(.*)/)?.at(-1); + .match(/(www\.)?(.*)/)?.at(-1); return hostname; } catch (error) { diff --git a/src/scrapers/newsensations.js b/src/scrapers/newsensations.js index 7bdc1f5e..0f84cfcd 100755 --- a/src/scrapers/newsensations.js +++ b/src/scrapers/newsensations.js @@ -1,90 +1,165 @@ 'use strict'; -const { geta, ed } = require('../utils/q'); +const unprint = require('unprint'); +const slugify = require('../utils/slugify'); -function scrapeBlockLatest(scenes) { - return scenes.map(({ html, qu }) => { +function scrapeLatestBlock(scenes) { + return scenes.map(({ html, query }) => { const release = {}; - const entryId = qu.q('div[class*="videothumb"]', 'class').match(/videothumb_(\d+)/) - || qu.q('div[id*="videothumb"]', 'id').match(/videothumb_(\d+)/); + release.title = query.content('h4 a'); + release.url = query.url('h4 a'); + release.date = unprint.extractDate(html, 'MM/DD/YYYY', { match: /\d{2}\/\d{2}\/\d{4}/ }); - release.entryId = entryId[1]; + release.actors = query.contents('.tour_update_models a'); - release.title = qu.q('h4 a', true); - release.url = qu.url('h4 a'); - release.date = ed(html, 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/); + // native videothumb entry ID does not occur on scene page, date is not available on all sites + release.entryId = slugify([...release.actors.sort(), release.title]); - release.actors = qu.all('.tour_update_models a', true); - - release.poster = qu.q('div img').dataset.src; - release.photos = [qu.q('div img', 'src0_4x') || qu.q('div img', 'src0_3x') || qu.q('div img', 'src0_2x')]; - - release.teaser = qu.video(); - - console.log(release); + release.poster = query.dataset('.video_placeholder', 'src'); + release.teaser = query.video(); return release; }); } -function scrapeClassicLatest(scenes) { - return scenes.map(({ el, qu }) => { +function scrapeLatestClassic(scenes) { + return scenes.map(({ query }) => { const release = {}; - release.entryId = el.dataset.setid; - release.url = qu.url('a'); + release.url = query.url('a'); - release.title = qu.q('.update_title_small', true) || qu.q('a:nth-child(2)', true); + release.title = query.content('.update_title_small') || query.content('a:nth-child(2)'); + release.description = query.attribute('a', 'title'); - const description = qu.q('a', 'title'); - if (description) release.description = description; + release.date = query.date('.date_small, .update_date', 'MM/DD/YYYY'); - const date = qu.date('.date_small, .update_date', 'MM/DD/YYYY'); - if (date) release.date = date; + release.duration = query.number('.update_counts') * 60; - const durationLine = qu.q('.update_counts', true); - if (durationLine) release.duration = Number(durationLine.match(/(\d+) min/i)[1]) * 60; + const actors = query.contents('.update_models a'); - const actors = qu.all('.update_models a', true); - release.actors = actors.length > 0 ? actors : qu.q('.update_models', true).split(/,\s*/); + release.actors = actors.length === 0 + ? query.content('.update_models').split(/,\s*/) + : actors; + + // native videothumb entry ID does not occur on scene page, date is not available on all sites + release.entryId = slugify([...release.actors.sort(), release.title]); + + const photoCount = query.number('.update_thumb', { attribute: 'cnt' }); - const photoCount = qu.q('.update_thumb', 'cnt'); [release.poster, ...release.photos] = Array.from({ length: photoCount }) - .map((value, index) => qu.q('.update_thumb', `src${index}_3x`) - || qu.q('.update_thumb', `src${index}_2x`) - || qu.q('.update_thumb', `src${index}_1x`)); + .map((value, index) => query.attribute('.update_thumb', `src${index}_3x`) + || query.attribute('.update_thumb', `src${index}_2x`) + || query.attribute('.update_thumb', `src${index}_1x`)); return release; }); } -function scrapeScene({ query }) { +function scrapeSceneBlock({ query }) { const release = {}; release.title = query.content('.indScene h2'); + release.description = query.content('.description'); + release.date = query.date('.sceneDateP span', 'MM/DD/YYYY'); + + release.actors = query.all('.sceneTextLink .tour_update_models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); + + release.duration = query.number('.sceneDateP', { match: /(\d+)\s+min/i, matchIndex: 1 }) * 60; + release.entryId = slugify([...release.actors.map((actor) => actor.name).sort(), release.title]); + + release.stars = query.number('.sceneRating'); + + release.poster = query.img('#trailer_thumb img[src*=content]'); + release.trailer = query.video('#trailerVideo source'); + + return release; +} + +async function fetchPhotos(url) { + const res = await unprint.get(url); + + if (res.ok) { + return res.context.query.imgs('.grid-gallery img'); + } + + return []; +} + +async function scrapeSceneClassic({ query }, context, options) { + const release = {}; + + release.title = query.content('.update_title'); + release.description = query.content('.update_description'); + release.date = query.date('.update_date', 'MM/DD/YYYY'); + + release.actors = query.all('.update_models a').map((actorEl) => ({ + name: unprint.query.content(actorEl), + url: unprint.query.url(actorEl, null), + })); + + release.entryId = slugify([...release.actors.map((actor) => actor.name).sort(), release.title]); + + release.tags = query.contents('.update_tags a'); + release.stars = query.number('.gallery_info', { match: /average\s+rating:\s+(\d+\.\d+)/i, matchIndex: 1 }); + + release.poster = query.img('#vidplayer', { attribute: 'poster' }); + release.trailer = query.video('#vidplayer source'); + + const gallery = query.url('//a[img[contains(@src, "gallery.gif")]]'); + + if (gallery && options.includePhotos) { + release.photos = await fetchPhotos(gallery); + } - console.log(release); return release; } async function fetchLatest(site, page = 1) { + const url = `${site.url}/tour_${site.parameters.siteId}/categories/movies_${page}_d.html`; + const res = await unprint.get(url, { selectAll: '.movieBlock, .videoBlock, .update_details, .update_details' }); + + return res; +} + +async function fetchLatestClassic(site, page) { if (!site.parameters) { return null; } - const url = `${site.url}/tour_${site.parameters.siteId}/categories/movies_${page}_d.html`; - const res = await geta(url, '.updatesBlock .movieBlock, .updatesBlock .videoBlock, .latest_updates_block .update_details, .category_listing_block .update_details'); + const res = await fetchLatest(site, page); - if (res.ok && site.parameters.block) { - return scrapeBlockLatest(res.items, site); + if (res.ok) { + return scrapeLatestClassic(res.context, site); } - return res.ok ? scrapeClassicLatest(res.items, site) : res.status; + return res.status; +} + +async function fetchLatestBlock(site, page) { + if (!site.parameters) { + return null; + } + + const res = await fetchLatest(site, page); + + if (res.ok) { + return scrapeLatestBlock(res.context, site); + } + + return res.status; } module.exports = { - fetchLatest, - scrapeScene, + fetchLatest: fetchLatestClassic, + scrapeScene: scrapeSceneClassic, useUnprint: true, + block: { + scrapeScene: scrapeSceneBlock, + fetchLatest: fetchLatestBlock, + useUnprint: true, + }, }; diff --git a/src/store-releases.js b/src/store-releases.js index 85775771..032789fa 100755 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -36,6 +36,7 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce url: release.url, date: Number(release.date) ? release.date : null, date_precision: release.datePrecision, + duration: Number(release.duration) || null, slug, description: release.description, comment: release.comment, diff --git a/src/utils/slugify.js b/src/utils/slugify.js index 46dc8801..ca0a3238 100755 --- a/src/utils/slugify.js +++ b/src/utils/slugify.js @@ -36,21 +36,24 @@ const substitutes = { ỹ: 'y', }; -function slugify(string, delimiter = '-', { +function slugify(strings, delimiter = '-', { encode = false, removeAccents = true, removePunctuation = false, limit = 1000, } = {}) { - if (!string || typeof string !== 'string') { - return string; + if (!strings || (typeof strings !== 'string' && !Array.isArray(strings))) { + return strings; } - const slugComponents = string - .trim() - .toLowerCase() - .replace(removePunctuation && /[.,:;'"_-]/g, '') - .match(/[A-Za-zÀ-ÖØ-öø-ÿ0-9]+/g); + const slugComponents = [] + .concat(strings) + .filter(Boolean) + .flatMap((string) => string + .trim() + .toLowerCase() + .replace(removePunctuation && /[.,:;'"_-]/g, '') + .match(/[A-Za-zÀ-ÖØ-öø-ÿ0-9]+/g)); if (!slugComponents) { return '';