diff --git a/public/img/logos/21naturals/favicon.png b/public/img/logos/21naturals/favicon.png new file mode 100644 index 00000000..abcec559 Binary files /dev/null and b/public/img/logos/21naturals/favicon.png differ diff --git a/public/img/logos/21naturals/network.png b/public/img/logos/21naturals/network.png new file mode 100644 index 00000000..58fda3e6 Binary files /dev/null and b/public/img/logos/21naturals/network.png differ diff --git a/public/img/logos/21sextreme/dominatedgirls.png b/public/img/logos/21sextreme/dominatedgirls.png new file mode 100644 index 00000000..8d99c11a Binary files /dev/null and b/public/img/logos/21sextreme/dominatedgirls.png differ diff --git a/public/img/logos/21sextreme/favicon.png b/public/img/logos/21sextreme/favicon.png new file mode 100644 index 00000000..096eeb49 Binary files /dev/null and b/public/img/logos/21sextreme/favicon.png differ diff --git a/public/img/logos/21sextreme/grandpasfuckteens.png b/public/img/logos/21sextreme/grandpasfuckteens.png new file mode 100644 index 00000000..a5906ce6 Binary files /dev/null and b/public/img/logos/21sextreme/grandpasfuckteens.png differ diff --git a/public/img/logos/21sextreme/homepornreality.png b/public/img/logos/21sextreme/homepornreality.png new file mode 100644 index 00000000..2d5c790b Binary files /dev/null and b/public/img/logos/21sextreme/homepornreality.png differ diff --git a/public/img/logos/21sextreme/lustygrandmas.png b/public/img/logos/21sextreme/lustygrandmas.png new file mode 100644 index 00000000..cc4f7d83 Binary files /dev/null and b/public/img/logos/21sextreme/lustygrandmas.png differ diff --git a/public/img/logos/21sextreme/mightymistress.png b/public/img/logos/21sextreme/mightymistress.png new file mode 100644 index 00000000..5fffaaf8 Binary files /dev/null and b/public/img/logos/21sextreme/mightymistress.png differ diff --git a/public/img/logos/21sextreme/misc/21sextreme.png b/public/img/logos/21sextreme/misc/21sextreme.png new file mode 100644 index 00000000..87177270 Binary files /dev/null and b/public/img/logos/21sextreme/misc/21sextreme.png differ diff --git a/public/img/logos/21sextreme/misc/21sextreme.svg b/public/img/logos/21sextreme/misc/21sextreme.svg new file mode 100644 index 00000000..64aadf18 --- /dev/null +++ b/public/img/logos/21sextreme/misc/21sextreme.svg @@ -0,0 +1,2480 @@ + + + + + + + + + + + 21sextury-logo + + + + + + + + + + + + + + + diff --git a/public/img/logos/21sextreme/misc/grandpasfuckteens_dark.png b/public/img/logos/21sextreme/misc/grandpasfuckteens_dark.png new file mode 100644 index 00000000..5a00fbc8 Binary files /dev/null and b/public/img/logos/21sextreme/misc/grandpasfuckteens_dark.png differ diff --git a/public/img/logos/21sextreme/misc/lusty-grandmas_dark.png b/public/img/logos/21sextreme/misc/lusty-grandmas_dark.png new file mode 100644 index 00000000..815262b7 Binary files /dev/null and b/public/img/logos/21sextreme/misc/lusty-grandmas_dark.png differ diff --git a/public/img/logos/21sextreme/network.png b/public/img/logos/21sextreme/network.png new file mode 100644 index 00000000..07f6a83d Binary files /dev/null and b/public/img/logos/21sextreme/network.png differ diff --git a/public/img/logos/21sextreme/oldyounglesbianlove.png b/public/img/logos/21sextreme/oldyounglesbianlove.png new file mode 100644 index 00000000..1ee3ceaa Binary files /dev/null and b/public/img/logos/21sextreme/oldyounglesbianlove.png differ diff --git a/public/img/logos/21sextreme/teachmefisting.png b/public/img/logos/21sextreme/teachmefisting.png new file mode 100644 index 00000000..0f4bf781 Binary files /dev/null and b/public/img/logos/21sextreme/teachmefisting.png differ diff --git a/public/img/logos/21sextreme/zoliboy.png b/public/img/logos/21sextreme/zoliboy.png new file mode 100644 index 00000000..93676d7f Binary files /dev/null and b/public/img/logos/21sextreme/zoliboy.png differ diff --git a/seeds/00_networks.js b/seeds/00_networks.js index f4e1c7f1..23303d9f 100644 --- a/seeds/00_networks.js +++ b/seeds/00_networks.js @@ -8,6 +8,18 @@ const networks = [ url: 'https://www.21sextury.com', description: 'Watch all the latest scenes and porn video updates on 21Sextury.com, the best European porn site with the hottest pornstars from all over the world! Watch porn videos from the large network here.', }, + { + slug: '21sextreme', + name: '21Sextreme', + url: 'https://www.21sextreme.com', + description: 'Welcome to 21Sextreme.com, your portal to fisting porn, old and young lesbians, horny grannies & extreme BDSM featuring the best Euro & American Pornstars', + }, + { + slug: '21naturals', + name: '21Naturals', + url: 'https://www.21naturals.com', + description: 'Welcome to 21Naturals.com, the porn network featuring the hottest pornstars from all over the world in all natural porn and erotic sex videos. Watch thousands of girls with natural tits', + }, { slug: 'adulttime', name: 'Adult Time', diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 5a10bb4d..14a38e1e 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -2,7 +2,74 @@ const upsert = require('../src/utils/upsert'); /* eslint-disable max-len */ const sites = [ - // 21Sextury + // 21SEXTREME + { + slug: 'grandpasfuckteens', + name: 'Grandpas Fuck Teens', + url: 'https://grandpasfuckteens.21sextreme.com', + network: '21sextreme', + }, + { + slug: 'oldyounglesbianlove', + name: 'Old Young Lesbian Love', + url: 'https://oldyounglesbianlove.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + { + slug: 'lustygrandmas', + name: 'Lusty Grandmas', + url: 'https://lustygrandmas.21sextreme.com', + network: '21sextreme', + }, + { + slug: 'teachmefisting', + name: 'Teach Me Fisting', + url: 'https://teachmefisting.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + { + slug: 'zoliboy', + name: 'Zoliboy', + url: 'https://zoliboy.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + { + slug: 'mightymistress', + name: 'Mighty Mistress', + url: 'https://mightymistress.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + { + slug: 'dominatedgirls', + name: 'Dominated Girls', + url: 'https://dominatedgirls.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + { + slug: 'homepornreality', + name: 'Home Porn Reality', + url: 'https://homepornreality.21sextreme.com', + network: '21sextreme', + parameters: { + scene: 'https://21sextreme.com/en/video', + }, + }, + // 21SEXTURY { slug: 'analteenangels', name: 'Anal Teen Angels', diff --git a/src/scrape-sites.js b/src/scrape-sites.js index 8cde55f9..2aa3ee4b 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -12,6 +12,14 @@ const { scrapeRelease } = require('./scrape-releases'); const { storeReleases } = require('./releases'); function getAfterDate() { + if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) { + // using date + return moment + .utc(argv.after, ['YYYY-MM-DD', 'DD-MM-YYYY']) + .toDate(); + } + + // using time distance (e.g. "1 month") return moment .utc() .subtract(...argv.after.split(' ')) @@ -83,11 +91,17 @@ async function deepFetchReleases(baseReleases) { try { const fullRelease = await scrapeRelease(release.url, release, 'scene'); - return { - ...release, - ...fullRelease, - deep: true, - }; + if (fullRelease) { + return { + ...release, + ...fullRelease, + deep: true, + }; + } + + logger.warn(`Release scraper returned empty result for ${release.url}`); + + return release; } catch (error) { logger.error(error.stack); diff --git a/src/scrapers/21naturals.js b/src/scrapers/21naturals.js new file mode 100644 index 00000000..8d972832 --- /dev/null +++ b/src/scrapers/21naturals.js @@ -0,0 +1,10 @@ +'use strict'; + +const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); + +module.exports = { + fetchLatest: fetchApiLatest, + fetchProfile: fetchApiProfile, + fetchUpcoming: fetchApiUpcoming, + fetchScene, +}; diff --git a/src/scrapers/21sextreme.js b/src/scrapers/21sextreme.js new file mode 100644 index 00000000..8d972832 --- /dev/null +++ b/src/scrapers/21sextreme.js @@ -0,0 +1,10 @@ +'use strict'; + +const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma'); + +module.exports = { + fetchLatest: fetchApiLatest, + fetchProfile: fetchApiProfile, + fetchUpcoming: fetchApiUpcoming, + fetchScene, +}; diff --git a/src/scrapers/brazzers.js b/src/scrapers/brazzers.js index 8f406a89..0eab0623 100644 --- a/src/scrapers/brazzers.js +++ b/src/scrapers/brazzers.js @@ -6,6 +6,8 @@ const cheerio = require('cheerio'); const { JSDOM } = require('jsdom'); const moment = require('moment'); +const { get, ex } = require('../utils/q'); +const slugify = require('../utils/slugify'); const { heightToCm, lbsToKg } = require('../utils/convert'); const hairMap = { @@ -15,7 +17,7 @@ const hairMap = { Redhead: 'red', }; -function scrape(html, site, upcoming) { +function scrapeAll(html, site, upcoming) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const sceneElements = $('.release-card.scene').toArray(); @@ -41,6 +43,8 @@ function scrape(html, site, upcoming) { const poster = `https:${$(element).find('.card-main-img').attr('data-src')}`; const photos = $(element).find('.card-overlay .image-under').map((photoIndex, photoElement) => `https:${$(photoElement).attr('data-src')}`).toArray(); + const channel = slugify($(element).find('.collection').attr('title'), { delimiter: '' }); + return acc.concat({ url, entryId, @@ -53,66 +57,55 @@ function scrape(html, site, upcoming) { likes, dislikes, }, + channel, site, }); }, []); } -async function scrapeScene(html, url, site) { +async function scrapeScene(html, url, _site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); + const release = {}; + const videoJson = $('script:contains("window.videoUiOptions")').html(); - const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('"},') + 2)); + const videoString = videoJson.slice(videoJson.indexOf('{"stream_info":'), videoJson.lastIndexOf('},') + 1); + const videoData = JSON.parse(videoString); - const entryId = url.split('/').slice(-3, -2)[0]; - const title = $('.scene-title[itemprop="name"]').text(); + [release.entryId] = url.split('/').slice(-3, -2); + release.title = $('.scene-title[itemprop="name"]').text(); - const description = $('#scene-description p[itemprop="description"]') + release.description = $('#scene-description p[itemprop="description"]') .contents() .first() .text() .trim(); - const date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate(); - const actors = $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); - const duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60; + release.date = moment.utc($('.more-scene-info .scene-date').text(), 'MMMM DD, YYYY').toDate(); + release.actors = $('.related-model a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); + release.duration = Number($('.scene-length[itemprop="duration"]').attr('content').slice(1, -1)) * 60; - const likes = Number($('.label-rating .like').text()); - const dislikes = Number($('.label-rating .dislike').text()); + release.likes = Number($('.label-rating .like').text()); + release.dislikes = Number($('.label-rating .dislike').text()); const siteElement = $('.niche-site-logo'); // const siteUrl = `https://www.brazzers.com${siteElement.attr('href').slice(0, -1)}`; const siteName = siteElement.attr('title'); - const channel = siteName.replace(/\s+/g, '').toLowerCase(); + release.channel = siteName.replace(/\s+/g, '').toLowerCase(); - const tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); + release.tags = $('.tag-card-container a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); + release.photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray(); - const poster = `https:${videoData.poster}`; - const photos = $('.carousel-thumb a').map((photoIndex, photoElement) => `https:${$(photoElement).attr('href')}`).toArray(); + const posterPath = videoData?.poster || $('meta[itemprop="thumbnailUrl"]').attr('content') || $('#trailer-player-container').attr('data-player-img'); + if (posterPath) release.poster = `https:${posterPath}`; - const trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({ - src: `https:${path}`, - quality: Number(quality.match(/\d{3,}/)[0]), - })); + if (videoData) { + release.trailer = Object.entries(videoData.stream_info.http.paths).map(([quality, path]) => ({ + src: `https:${path}`, + quality: Number(quality.match(/\d{3,}/)[0]), + })); + } - return { - url, - entryId, - title, - description, - actors, - date, - poster, - photos, - trailer, - duration, - rating: { - likes, - dislikes, - }, - tags, - site, - channel, - }; + return release; } function scrapeActorSearch(html, url, actorName) { @@ -122,13 +115,26 @@ function scrapeActorSearch(html, url, actorName) { return actorLink ? actorLink.href : null; } -function scrapeProfile(html, url, actorName) { - const { document } = new JSDOM(html).window; +async function fetchActorReleases({ qu, html }, accReleases = []) { + const releases = scrapeAll(html); + const next = qu('.pagination .next a'); - const avatarEl = document.querySelector('.big-pic-model-container img'); - const descriptionEl = document.querySelector('.model-profile-specs p'); - const bioKeys = Array.from(document.querySelectorAll('.profile-spec-list label'), el => el.textContent.replace(/\n+|\s{2,}/g, '').trim()); - const bioValues = Array.from(document.querySelectorAll('.profile-spec-list var'), el => el.textContent.replace(/\n+|\s{2,}/g, '').trim()); + if (next) { + const url = `https://www.brazzers.com${next}`; + const qNext = await get(url); + + return fetchActorReleases(qNext, accReleases.concat(releases)); + } + + return accReleases.concat(releases); +} + +async function scrapeProfile(html, url, actorName) { + const qProfile = ex(html); + const { q, qa } = qProfile; + + const bioKeys = qa('.profile-spec-list label', true).map(key => key.replace(/\n+|\s{2,}/g, '').trim()); + const bioValues = qa('.profile-spec-list var', true).map(value => value.replace(/\n+|\s{2,}/g, '').trim()); const bio = bioKeys.reduce((acc, key, index) => ({ ...acc, [key]: bioValues[index] }), {}); @@ -136,6 +142,8 @@ function scrapeProfile(html, url, actorName) { name: actorName, }; + profile.description = q('.model-profile-specs p', true); + if (bio.Ethnicity) profile.ethnicity = bio.Ethnicity; if (bio.Measurements && bio.Measurements.match(/\d+[A-Z]+-\d+-\d+/)) [profile.bust, profile.waist, profile.hip] = bio.Measurements.split('-'); if (bio['Date of Birth'] && bio['Date of Birth'] !== 'Unknown') profile.birthdate = moment.utc(bio['Date of Birth'], 'MMMM DD, YYYY').toDate(); @@ -152,10 +160,10 @@ function scrapeProfile(html, url, actorName) { if (bio['Body Art'] && bio['Body Art'].match('Tattoo')) profile.hasTattoos = true; if (bio['Body Art'] && bio['Body Art'].match('Piercing')) profile.hasPiercings = true; - if (descriptionEl) profile.description = descriptionEl.textContent.trim(); + const avatarEl = q('.big-pic-model-container img'); if (avatarEl) profile.avatar = `https:${avatarEl.src}`; - profile.releases = Array.from(document.querySelectorAll('.release-card-container .scene-card-title a'), el => `https://brazzers.com${el.href}`); + profile.releases = await fetchActorReleases(qProfile); return profile; } @@ -163,13 +171,13 @@ function scrapeProfile(html, url, actorName) { async function fetchLatest(site, page = 1) { const res = await bhttp.get(`${site.url}/page/${page}/`); - return scrape(res.body.toString(), site, false); + return scrapeAll(res.body.toString(), site, false); } async function fetchUpcoming(site) { const res = await bhttp.get(`${site.url}/`); - return scrape(res.body.toString(), site, true); + return scrapeAll(res.body.toString(), site, true); } async function fetchScene(url, site) { diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index a7ae21bb..4174ddd1 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -186,7 +186,7 @@ async function scrapeScene(html, url, site) { // date in data object is not the release date of the scene, but the date the entry was added; only use as fallback const dateString = $('.updatedDate').first().text().trim(); - const dateMatch = dateString.match(/\d{2,4}-\d{2}-\d{2,4}/)?.[0]; + const dateMatch = dateString.match(/\d{2,4}[-/]\d{2}[-/]\d{2,4}/)?.[0]; if (dateMatch) release.date = moment.utc(dateMatch, ['MM-DD-YYYY', 'YYYY-MM-DD']).toDate(); else if (data?.dateCreated) release.date = moment.utc(data.dateCreated, 'YYYY-MM-DD').toDate(); diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index b70e1b32..286dbf92 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -16,6 +16,7 @@ const famedigital = require('./famedigital'); const fantasymassage = require('./fantasymassage'); const freeones = require('./freeones'); const freeonesLegacy = require('./freeones_legacy'); +const girlsway = require('./girlsway'); const iconmale = require('./iconmale'); const jayrock = require('./jayrock'); const julesjordan = require('./julesjordan'); @@ -28,7 +29,7 @@ const mikeadriano = require('./mikeadriano'); const milehighmedia = require('./milehighmedia'); const mindgeek = require('./mindgeek'); const mofos = require('./mofos'); -const girlsway = require('./girlsway'); +const naturals = require('./21naturals'); const naughtyamerica = require('./naughtyamerica'); const perfectgonzo = require('./perfectgonzo'); const pervcity = require('./pervcity'); @@ -38,9 +39,10 @@ const privateNetwork = require('./private'); // reserved keyword const puretaboo = require('./puretaboo'); const realitykings = require('./realitykings'); const score = require('./score'); +const sextreme = require('./21sextreme'); +const sextury = require('./21sextury'); const teamskeet = require('./teamskeet'); const transangels = require('./transangels'); -const twentyonesextury = require('./21sextury'); const twistys = require('./twistys'); const vixen = require('./vixen'); const vogov = require('./vogov'); @@ -50,7 +52,9 @@ const xempire = require('./xempire'); module.exports = { releases: { adulttime, - '21sextury': twentyonesextury, + '21naturals': naturals, + '21sextreme': sextreme, + '21sextury': sextury, babes, bang, bangbros, @@ -93,7 +97,7 @@ module.exports = { }, actors: { // ordered by data priority - '21sextury': twentyonesextury, + '21sextury': sextury, babes, bangbros, blowpass,