diff --git a/config/default.js b/config/default.js index 7cbf23f3..272973b5 100644 --- a/config/default.js +++ b/config/default.js @@ -22,7 +22,7 @@ module.exports = { 'famedigital', ], [ - // Gamma; Evil Angel + Devil's Film and Wicked have their own assets + // Gamma; Evil Angel + Devil's Film, Pure Taboo (unavailable) and Wicked have their own assets 'xempire', 'blowpass', ], diff --git a/public/img/logos/fantasymassage/allgirlmassage.png b/public/img/logos/fantasymassage/allgirlmassage.png new file mode 100644 index 00000000..294482df Binary files /dev/null and b/public/img/logos/fantasymassage/allgirlmassage.png differ diff --git a/public/img/logos/fantasymassage/favicon.png b/public/img/logos/fantasymassage/favicon.png new file mode 100644 index 00000000..d8c398bd Binary files /dev/null and b/public/img/logos/fantasymassage/favicon.png differ diff --git a/public/img/logos/fantasymassage/misc/all-girl-massage.png b/public/img/logos/fantasymassage/misc/all-girl-massage.png new file mode 100644 index 00000000..3c7e1fd4 Binary files /dev/null and b/public/img/logos/fantasymassage/misc/all-girl-massage.png differ diff --git a/public/img/logos/fantasymassage/misc/all-girl-massage.svg b/public/img/logos/fantasymassage/misc/all-girl-massage.svg new file mode 100644 index 00000000..f1dff6e2 --- /dev/null +++ b/public/img/logos/fantasymassage/misc/all-girl-massage.svg @@ -0,0 +1,84 @@ + + + + +agm-hor + + + diff --git a/public/img/logos/fantasymassage/misc/all-girl-massage_square.png b/public/img/logos/fantasymassage/misc/all-girl-massage_square.png new file mode 100644 index 00000000..5eb591f9 Binary files /dev/null and b/public/img/logos/fantasymassage/misc/all-girl-massage_square.png differ diff --git a/public/img/logos/fantasymassage/misc/fantasy-massage_square.png b/public/img/logos/fantasymassage/misc/fantasy-massage_square.png new file mode 100644 index 00000000..c070bb80 Binary files /dev/null and b/public/img/logos/fantasymassage/misc/fantasy-massage_square.png differ diff --git a/public/img/logos/fantasymassage/misc/nuru-massage.png b/public/img/logos/fantasymassage/misc/nuru-massage.png new file mode 100644 index 00000000..df27b8d7 Binary files /dev/null and b/public/img/logos/fantasymassage/misc/nuru-massage.png differ diff --git a/public/img/logos/fantasymassage/misc/nuru-massage.svg b/public/img/logos/fantasymassage/misc/nuru-massage.svg new file mode 100644 index 00000000..a8e7b740 --- /dev/null +++ b/public/img/logos/fantasymassage/misc/nuru-massage.svg @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/fantasymassage/misc/nuru-massage_square.png b/public/img/logos/fantasymassage/misc/nuru-massage_square.png new file mode 100644 index 00000000..8199eded Binary files /dev/null and b/public/img/logos/fantasymassage/misc/nuru-massage_square.png differ diff --git a/public/img/logos/fantasymassage/misc/tricky-spa_original.png b/public/img/logos/fantasymassage/misc/tricky-spa_original.png new file mode 100644 index 00000000..c883b18c Binary files /dev/null and b/public/img/logos/fantasymassage/misc/tricky-spa_original.png differ diff --git a/public/img/logos/fantasymassage/network.png b/public/img/logos/fantasymassage/network.png new file mode 100644 index 00000000..4df019c7 Binary files /dev/null and b/public/img/logos/fantasymassage/network.png differ diff --git a/public/img/logos/fantasymassage/nurumassage.png b/public/img/logos/fantasymassage/nurumassage.png new file mode 100644 index 00000000..aef8a086 Binary files /dev/null and b/public/img/logos/fantasymassage/nurumassage.png differ diff --git a/public/img/logos/fantasymassage/trickyspa.png b/public/img/logos/fantasymassage/trickyspa.png new file mode 100644 index 00000000..65155c3d Binary files /dev/null and b/public/img/logos/fantasymassage/trickyspa.png differ diff --git a/public/img/logos/girlsway/favicon.png b/public/img/logos/girlsway/favicon.png new file mode 100644 index 00000000..2adfb913 Binary files /dev/null and b/public/img/logos/girlsway/favicon.png differ diff --git a/public/img/logos/girlsway/girlstryanal.png b/public/img/logos/girlsway/girlstryanal.png new file mode 100644 index 00000000..0df0b642 Binary files /dev/null and b/public/img/logos/girlsway/girlstryanal.png differ diff --git a/public/img/logos/adulttime/girlsway.png b/public/img/logos/girlsway/girlsway.png similarity index 99% rename from public/img/logos/adulttime/girlsway.png rename to public/img/logos/girlsway/girlsway.png index a7e668f9..d65a7692 100644 Binary files a/public/img/logos/adulttime/girlsway.png and b/public/img/logos/girlsway/girlsway.png differ diff --git a/public/img/logos/girlsway/misc/girls-try-anal.png b/public/img/logos/girlsway/misc/girls-try-anal.png new file mode 100644 index 00000000..2f2a9a53 Binary files /dev/null and b/public/img/logos/girlsway/misc/girls-try-anal.png differ diff --git a/public/img/logos/girlsway/misc/girls-try-anal.svg b/public/img/logos/girlsway/misc/girls-try-anal.svg new file mode 100644 index 00000000..b67b6d46 --- /dev/null +++ b/public/img/logos/girlsway/misc/girls-try-anal.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/adulttime/misc/girls-way.png b/public/img/logos/girlsway/misc/girls-way.png similarity index 100% rename from public/img/logos/adulttime/misc/girls-way.png rename to public/img/logos/girlsway/misc/girls-way.png diff --git a/public/img/logos/adulttime/misc/girls-way.svg b/public/img/logos/girlsway/misc/girls-way.svg similarity index 100% rename from public/img/logos/adulttime/misc/girls-way.svg rename to public/img/logos/girlsway/misc/girls-way.svg diff --git a/public/img/logos/girlsway/misc/mommys-girl.png b/public/img/logos/girlsway/misc/mommys-girl.png new file mode 100644 index 00000000..9f8efa3a Binary files /dev/null and b/public/img/logos/girlsway/misc/mommys-girl.png differ diff --git a/public/img/logos/girlsway/misc/mommys-girl.svg b/public/img/logos/girlsway/misc/mommys-girl.svg new file mode 100644 index 00000000..e69c9d59 --- /dev/null +++ b/public/img/logos/girlsway/misc/mommys-girl.svg @@ -0,0 +1 @@ +mommysgirl-logo \ No newline at end of file diff --git a/public/img/logos/girlsway/misc/web-young.png b/public/img/logos/girlsway/misc/web-young.png new file mode 100644 index 00000000..890ab75f Binary files /dev/null and b/public/img/logos/girlsway/misc/web-young.png differ diff --git a/public/img/logos/girlsway/misc/web-young.svg b/public/img/logos/girlsway/misc/web-young.svg new file mode 100644 index 00000000..2f2687cf --- /dev/null +++ b/public/img/logos/girlsway/misc/web-young.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/girlsway/mommysgirl.png b/public/img/logos/girlsway/mommysgirl.png new file mode 100644 index 00000000..ac7010b4 Binary files /dev/null and b/public/img/logos/girlsway/mommysgirl.png differ diff --git a/public/img/logos/girlsway/network.png b/public/img/logos/girlsway/network.png new file mode 100644 index 00000000..362e8c65 Binary files /dev/null and b/public/img/logos/girlsway/network.png differ diff --git a/public/img/logos/girlsway/sextapelesbians.png b/public/img/logos/girlsway/sextapelesbians.png new file mode 100644 index 00000000..4d086cf9 Binary files /dev/null and b/public/img/logos/girlsway/sextapelesbians.png differ diff --git a/public/img/logos/girlsway/webyoung.png b/public/img/logos/girlsway/webyoung.png new file mode 100644 index 00000000..1fa82104 Binary files /dev/null and b/public/img/logos/girlsway/webyoung.png differ diff --git a/public/img/logos/puretaboo/favicon.png b/public/img/logos/puretaboo/favicon.png index adaba32b..9fe921a0 100644 Binary files a/public/img/logos/puretaboo/favicon.png and b/public/img/logos/puretaboo/favicon.png differ diff --git a/public/img/logos/puretaboo/futuredarkly.png b/public/img/logos/puretaboo/futuredarkly.png new file mode 100644 index 00000000..5688e36d Binary files /dev/null and b/public/img/logos/puretaboo/futuredarkly.png differ diff --git a/public/img/logos/puretaboo/lesbianrevenge.png b/public/img/logos/puretaboo/lesbianrevenge.png new file mode 100644 index 00000000..27108af3 Binary files /dev/null and b/public/img/logos/puretaboo/lesbianrevenge.png differ diff --git a/public/img/logos/puretaboo/misc/future-darkly.png b/public/img/logos/puretaboo/misc/future-darkly.png new file mode 100644 index 00000000..b942adf3 Binary files /dev/null and b/public/img/logos/puretaboo/misc/future-darkly.png differ diff --git a/public/img/logos/puretaboo/misc/future-darkly.svg b/public/img/logos/puretaboo/misc/future-darkly.svg new file mode 100644 index 00000000..987d911b --- /dev/null +++ b/public/img/logos/puretaboo/misc/future-darkly.svg @@ -0,0 +1,38 @@ + + + + + + diff --git a/public/img/logos/puretaboo/misc/lesbian-revenge.png b/public/img/logos/puretaboo/misc/lesbian-revenge.png new file mode 100644 index 00000000..ecdd4cd4 Binary files /dev/null and b/public/img/logos/puretaboo/misc/lesbian-revenge.png differ diff --git a/public/img/logos/puretaboo/misc/lesbian-revenge.svg b/public/img/logos/puretaboo/misc/lesbian-revenge.svg new file mode 100644 index 00000000..fd41ec65 --- /dev/null +++ b/public/img/logos/puretaboo/misc/lesbian-revenge.svg @@ -0,0 +1,1781 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/puretaboo/network.png b/public/img/logos/puretaboo/network.png index fc514ec0..1b511c46 100644 Binary files a/public/img/logos/puretaboo/network.png and b/public/img/logos/puretaboo/network.png differ diff --git a/public/img/logos/puretaboo/prettydirty.png b/public/img/logos/puretaboo/prettydirty.png new file mode 100644 index 00000000..285b32e3 Binary files /dev/null and b/public/img/logos/puretaboo/prettydirty.png differ diff --git a/public/img/logos/puretaboo/puretaboo.png b/public/img/logos/puretaboo/puretaboo.png index 6a54691e..a66dba98 100644 Binary files a/public/img/logos/puretaboo/puretaboo.png and b/public/img/logos/puretaboo/puretaboo.png differ diff --git a/public/img/logos/adulttime/underthebed.png b/public/img/logos/puretaboo/underthebed.png similarity index 98% rename from public/img/logos/adulttime/underthebed.png rename to public/img/logos/puretaboo/underthebed.png index 1a773f86..4d155ea3 100644 Binary files a/public/img/logos/adulttime/underthebed.png and b/public/img/logos/puretaboo/underthebed.png differ diff --git a/seeds/00_networks.js b/seeds/00_networks.js index b9a12a9c..f4e1c7f1 100644 --- a/seeds/00_networks.js +++ b/seeds/00_networks.js @@ -79,12 +79,24 @@ const networks = [ url: 'https://evilangel.com', description: 'Welcome to the award winning Evil Angel website, home to the most popular pornstars of today, yesterday and tomorrow in their most extreme and hardcore porn scenes to date. We feature almost 30 years of rough sex videos and hardcore anal porn like you\'ve never seen before, and have won countless AVN and XBiz awards including \'Best Site\' and \'Best Studio\'.', }, + { + slug: 'fantasymassage', + name: 'Fantasy Massage', + url: 'https://www.fantasymassage.com', + description: '', + }, { slug: 'fakehub', name: 'Fake Hub', url: 'https://www.fakehub.com', description: 'Wherever they go, there is porn. Hospital, Taxis, Casting… Maybe fucking to a fake cop, fake agent or fake taxi driver. And we record it all.', }, + { + slug: 'girlsway', + name: 'Girlsway', + url: 'https://www.girlsway.com', + description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!', + }, { slug: 'jayrock', name: 'JayRock Productions', diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 259d437e..0f3e13d9 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -137,27 +137,6 @@ const sites = [ description: 'LadyGonzo.com is a new Adult Time porn series featuring Joanna Angel shooting hardcore sex and gonzo porn movies the way she\'d like to see it!', network: 'adulttime', }, - { - name: 'Girlsway', - slug: 'girlsway', - url: 'https://www.girlsway.com', - description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!', - network: 'adulttime', - parameters: { - scene: 'https://www.girlsway.com/en/video/girlsway', - }, - }, - { - name: 'Under The Bed', - slug: 'underthebed', - url: 'https://www.underthebed.com', - description: '', - network: 'adulttime', - parameters: { - referer: 'https://freetour.underthebed.com/en/join', - deep: false, - }, - }, // BABES { name: 'Babes', @@ -1477,6 +1456,93 @@ const sites = [ upcoming: '/en/scenes/All/0/superCat/0/upcoming', }, }, + // FANTASY MASSAGE + { + slug: 'allgirlmassage', + name: 'All Girl Massage', + url: 'https://www.allgirlmassage.com', + network: 'fantasymassage', + parameters: { + latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, + }, + { + slug: 'nurumassage', + name: 'Nuru Massage', + url: 'https://www.nurumassage.com', + network: 'fantasymassage', + parameters: { + latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + }, + }, + { + slug: 'trickyspa', + name: 'Tricky Spa', + url: 'https://www.trickyspa.com', + network: 'fantasymassage', + parameters: { + latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + photos: '/en/photogallery/', + }, + }, + { + slug: 'soapymassage', + name: 'Soapy Massage', + url: 'https://www.soapymassage.com', + network: 'fantasymassage', + parameters: { + latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/', + upcoming: '/en/videos/upcoming', + photos: '/en/photogallery/', + }, + }, + // GIRLSWAY + { + slug: 'girlsway', + name: 'Girlsway', + url: 'https://www.girlsway.com', + description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!', + network: 'girlsway', + parameters: { + scene: 'https://www.girlsway.com/en/video/girlsway', + }, + }, + { + slug: 'girlstryanal', + name: 'Girls Try Anal', + url: 'https://www.girlstryanal.com', + network: 'girlsway', + parameters: { + referer: 'https://www.girlsway.com', + }, + }, + { + slug: 'mommysgirl', + name: 'Mommy\'s Girl', + url: 'https://www.mommysgirl.com', + network: 'girlsway', + }, + { + slug: 'webyoung', + name: 'Web Young', + url: 'https://www.webyoung.com', + network: 'girlsway', + parameters: { + referer: 'https://www.girlsway.com', + }, + }, + { + slug: 'sextapelesbians', + name: 'Sex Tape Lesbians', + url: 'https://www.sextapelesbians.com', + network: 'girlsway', + parameters: { + referer: 'https://www.girlsway.com', + }, + }, // JAYS POV { slug: 'jayspov', @@ -2797,13 +2863,54 @@ const sites = [ }, // PURE TABOO { - slug: 'puretaboo', name: 'Pure Taboo', + slug: 'puretaboo', url: 'https://www.puretaboo.com', description: 'PureTaboo.com is the ultimate site for family taboo porn, featuring submissive teens & virgins in rough sex videos in ultra 4k HD.', + network: 'puretaboo', + priority: 1, parameters: { independent: true }, + }, + { + name: 'Pretty Dirty', + slug: 'prettydirty', + url: 'https://www.prettydirty.com', + network: 'puretaboo', + parameters: { + referer: 'https://www.puretaboo.com', + }, + }, + /* series, not sites, that appear on Pure Taboo itself + { + name: 'Under The Bed', + slug: 'underthebed', + url: 'https://www.underthebed.com', + description: '', + network: 'puretaboo', + parameters: { + scene: 'https://www.puretaboo.com/en/video', + referer: 'https://www.puretaboo.com', + }, + }, + { + name: 'Lesbian Revenge', + slug: 'lesbianrevenge', + url: 'https://www.lesbianrevenge.com', + description: '', + network: 'puretaboo', + parameters: { + scene: 'https://www.puretaboo.com/en/video', + referer: 'https://www.puretaboo.com', + }, + }, + { + name: 'Future Darkly', + slug: 'futuredarkly', + url: 'https://www.futuredarkly.com', + description: '', network: 'puretaboo', }, + */ // REALITY KINGS { name: 'Look At Her Now', diff --git a/src/actors.js b/src/actors.js index 61ebbd66..c5024b5a 100644 --- a/src/actors.js +++ b/src/actors.js @@ -5,7 +5,7 @@ const Promise = require('bluebird'); const UrlPattern = require('url-pattern'); const moment = require('moment'); -const logger = require('./logger'); +const logger = require('./logger')(__filename); const knex = require('./knex'); const argv = require('./argv'); const scrapers = require('./scrapers/scrapers'); diff --git a/src/logger.js b/src/logger.js index a144b2f8..7e78981c 100644 --- a/src/logger.js +++ b/src/logger.js @@ -1,25 +1,30 @@ 'use strict'; const util = require('util'); +const path = require('path'); const winston = require('winston'); -const args = require('./argv'); + require('winston-daily-rotate-file'); -function logger(filename) { +const args = require('./argv'); + +function logger(filepath) { + const root = filepath.match(/src\/|dist\//); + const filename = filepath.slice(root.index + root[0].length) + .replace(path.extname(filepath), ''); + return winston.createLogger({ format: winston.format.combine( - winston.format.timestamp(), - winston.format.printf(info => `${info.timestamp} [${info.label}] ${info.level}: ${util.inspect(info.message)}`), + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + winston.format(info => (info instanceof Error + ? { ...info, message: info.stack } + : { ...info, message: typeof info.message === 'string' ? info.message : util.inspect(info.message) }))(), + winston.format.colorize(), + winston.format.printf(({ level, timestamp, label, message }) => `${timestamp} ${level} [${label || filename}] ${message}`), ), transports: [ new winston.transports.Console({ level: args.level, - format: winston.format.combine( - winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), - winston.format.colorize(), - // winston.format.printf(info => `${info.timestamp} ${info.level}: ${info.message}`), - winston.format.printf(info => `${info.timestamp} ${info.level} [${filename}]: ${util.inspect(info.message)}`), - ), timestamp: true, }), new winston.transports.DailyRotateFile({ diff --git a/src/releases.js b/src/releases.js index 54f5a6bb..a55b3488 100644 --- a/src/releases.js +++ b/src/releases.js @@ -4,7 +4,7 @@ const config = require('config'); const Promise = require('bluebird'); const moment = require('moment'); -const logger = require('./logger'); +const logger = require('./logger')(__filename); const knex = require('./knex'); const argv = require('./argv'); const whereOr = require('./utils/where-or'); @@ -358,7 +358,7 @@ async function storeReleaseAssets(releases) { role: 'teaser', }, identifier); } catch (error) { - logger.error({ label: 'media', message: error.message }); + logger.error(error.message); } }, { concurrency: 10, @@ -422,7 +422,7 @@ async function storeReleases(releases) { ...releaseWithChannelSite, }; } catch (error) { - logger.error({ label: 'media', message: error.message }); + logger.error(error); return null; } diff --git a/src/scrape-releases.js b/src/scrape-releases.js index 512393ac..cc19a8b8 100644 --- a/src/scrape-releases.js +++ b/src/scrape-releases.js @@ -3,7 +3,7 @@ const config = require('config'); const Promise = require('bluebird'); -const logger = require('./logger'); +const logger = require('./logger')(__filename); const argv = require('./argv'); const scrapers = require('./scrapers/scrapers'); const { findSiteByUrl } = require('./sites'); diff --git a/src/scrape-sites.js b/src/scrape-sites.js index b57baaf6..8cde55f9 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -4,7 +4,7 @@ const Promise = require('bluebird'); const moment = require('moment'); const argv = require('./argv'); -const logger = require('./logger'); +const logger = require('./logger')(__filename); const knex = require('./knex'); const { fetchIncludedSites } = require('./sites'); const scrapers = require('./scrapers/scrapers'); diff --git a/src/scrapers/adulttime.js b/src/scrapers/adulttime.js index d6a1cfa1..92fc03a1 100644 --- a/src/scrapers/adulttime.js +++ b/src/scrapers/adulttime.js @@ -22,8 +22,6 @@ async function networkFetchScene(url, site) { async function fetchLatest(site, page = 1) { const releases = await fetchApiLatest(site, page, false); - console.log(releases); - return releases.map(release => curateRelease(release, site)); } diff --git a/src/scrapers/fantasymassage.js b/src/scrapers/fantasymassage.js new file mode 100644 index 00000000..448e1646 --- /dev/null +++ b/src/scrapers/fantasymassage.js @@ -0,0 +1,10 @@ +'use strict'; + +const { fetchLatest, fetchUpcoming, fetchScene } = require('./gamma'); + + +module.exports = { + fetchLatest, + fetchScene, + fetchUpcoming, +}; diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 5137ada4..1ae11269 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -7,7 +7,7 @@ const cheerio = require('cheerio'); const moment = require('moment'); const argv = require('../argv'); -const logger = require('../logger'); +const logger = require('../logger')(__filename); const { ex, get } = require('../utils/q'); const slugify = require('../utils/slugify'); @@ -20,11 +20,12 @@ async function fetchPhotos(url) { function scrapePhotos(html) { const $ = cheerio.load(html, { normalizeWhitespace: true }); - return $('.preview .imgLink').toArray().map((linkEl) => { + return $('.preview .imgLink, .pgFooterThumb a').toArray().map((linkEl) => { const url = $(linkEl).attr('href'); - if (url.match('/join')) { + if (/\/join|\/createaccount/.test(url)) { // URL links to join page instead of full photo, extract thumbnail + // /createaccount is used by e.g. Tricky Spa const src = $(linkEl).find('img').attr('src'); if (src.match('previews/')) { @@ -46,12 +47,14 @@ function scrapePhotos(html) { } async function getPhotos(albumPath, site) { - const albumUrl = `${site.url}${albumPath}`; + const albumUrl = site.parameters?.photos + ? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}` + : `${site.url}${albumPath}`; try { const html = await fetchPhotos(albumUrl); const $ = cheerio.load(html, { normalizeWhitespace: true }); - const photos = scrapePhotos(html); + const photos = scrapePhotos(html, site); const lastPage = $('.Gamma_Paginator a.last').attr('href')?.match(/\d+$/)[0]; @@ -62,7 +65,7 @@ async function getPhotos(albumPath, site) { const pageUrl = `${site.url}/${albumPath}/${page}`; const pageHtml = await fetchPhotos(pageUrl); - return scrapePhotos(pageHtml); + return scrapePhotos(pageHtml, site); }, { concurrency: 2, }); @@ -97,9 +100,9 @@ async function scrapeApiReleases(json, site) { release.actors = scene.actors.map(({ name }) => name); release.director = scene.directors[0].name; - console.log(release.url); - - release.tags = scene.master_categories.concat(scene.categories?.map(category => category.name)); + release.tags = scene.master_categories + .concat(scene.categories?.map(category => category.name)) + .filter(Boolean); // some categories don't have a name const posterPath = scene.pictures.resized || (scene.pictures.nsfw?.top && Object.values(scene.pictures.nsfw.top)[0]); @@ -118,38 +121,35 @@ async function scrapeApiReleases(json, site) { function scrapeAll(html, site, networkUrl, hasTeaser = true) { const $ = cheerio.load(html, { normalizeWhitespace: true }); - const scenesElements = $('li[data-itemtype=scene]').toArray(); + const scenesElements = $('li[data-itemtype=scene], div[data-itemtype=scenes]').toArray(); return scenesElements.map((element) => { const release = {}; - const sceneLinkElement = $(element).find('.sceneTitle a'); + const sceneLinkElement = $(element).find('.sceneTitle a, .tlcTitle a'); if (site) release.url = `${networkUrl ? site.network.url : site.url}${sceneLinkElement.attr('href')}`; else release.url = `${networkUrl}${sceneLinkElement.attr('href')}`; release.title = sceneLinkElement.attr('title'); - release.entryId = $(element).attr('data-itemid'); - const dateEl = $(element).find('.sceneDate').text() || null; + const dateEl = $(element).find('.sceneDate, .tlcSpecsDate .tlcDetailsValue').text() || null; if (dateEl) { release.date = moment - .utc($(element).find('.sceneDate').text(), ['MM-DD-YYYY', 'YYYY-MM-DD']) + .utc(dateEl, ['MM-DD-YYYY', 'YYYY-MM-DD']) .toDate(); } - release.actors = $(element).find('.sceneActors a') + release.actors = $(element).find('.sceneActors a, .tlcActors a') .map((actorIndex, actorElement) => $(actorElement).attr('title')) .toArray(); - const [likes, dislikes] = $(element).find('.value') + [release.likes, release.dislikes] = $(element).find('.value') .toArray() .map(value => Number($(value).text())); - release.rating = { likes, dislikes }; - - const posterEl = $(element).find('.imgLink img'); + const posterEl = $(element).find('.imgLink img, .tlcImageItem'); if (posterEl) release.poster = posterEl.attr('data-original') || posterEl.attr('src'); if (hasTeaser) { @@ -189,18 +189,18 @@ async function scrapeScene(html, url, site) { release.director = data.director?.[0].name || data2?.director?.[0].name; const actors = data?.actor || data2?.actor || []; - const hasTrans = actors.some(actor => actor.gender === 'shemale'); release.actors = actors.map(actor => actor.name); const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5; if (stars) release.rating = { stars }; release.duration = moment.duration(data.duration.slice(2)).asSeconds(); - - const rawTags = data.keywords?.split(', '); - release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; } + const hasTrans = release.actors.some(actor => actor.gender === 'shemale'); + const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', '); + release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; + const channel = data?.productionCompany?.name || $('.studioLink a').attr('title')?.trim(); if (channel) release.channel = slugify(channel, { delimiter: '' }); @@ -345,7 +345,7 @@ function getApiUrl(appId, apiKey) { } async function fetchApiCredentials(referer, site) { - if (site.parameters?.appId && site.parameters?.apiKey) { + if (site?.parameters?.appId && site?.parameters?.apiKey) { return getApiUrl(site.parameters.appId, site.parameters.apiKey); } @@ -374,7 +374,7 @@ async function fetchApiLatest(site, page = 1, upcoming = false) { requests: [ { indexName: 'all_scenes', - params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug}`, + params: `query=&hitsPerPage=36&maxValuesPerFacet=100&page=${page - 1}&facetFilters=[["lesbian:"],["bisex:"],["shemale:"],["upcoming:${upcoming ? 1 : 0}"]]&filters=sitename:${site.slug} OR channels.id:${site.slug}`, }, ], }, { @@ -384,6 +384,8 @@ async function fetchApiLatest(site, page = 1, upcoming = false) { encodeJSON: true, }); + console.log(res.body); + if (res.statusCode === 200 && res.body.results?.[0]?.hits) { return scrapeApiReleases(res.body.results[0].hits, site); } diff --git a/src/scrapers/girlsway.js b/src/scrapers/girlsway.js new file mode 100644 index 00000000..151bb845 --- /dev/null +++ b/src/scrapers/girlsway.js @@ -0,0 +1,10 @@ +'use strict'; + +const { fetchApiLatest, fetchApiUpcoming, fetchScene } = require('./gamma'); + + +module.exports = { + fetchLatest: fetchApiLatest, + fetchScene, + fetchUpcoming: fetchApiUpcoming, +}; diff --git a/src/scrapers/jayrock.js b/src/scrapers/jayrock.js index 88d6816a..d481acc9 100644 --- a/src/scrapers/jayrock.js +++ b/src/scrapers/jayrock.js @@ -4,7 +4,7 @@ const Promise = require('bluebird'); const bhttp = require('bhttp'); -const logger = require('../logger'); +const logger = require('../logger')(__filename); const slugify = require('../utils/slugify'); async function fetchToken(site) { diff --git a/src/scrapers/puretaboo.js b/src/scrapers/puretaboo.js index dc95bc5a..151bb845 100644 --- a/src/scrapers/puretaboo.js +++ b/src/scrapers/puretaboo.js @@ -1,103 +1,10 @@ 'use strict'; -const bhttp = require('bhttp'); -const moment = require('moment'); +const { fetchApiLatest, fetchApiUpcoming, fetchScene } = require('./gamma'); -// const { fetchLatest } = require('./gamma'); -const { ex, ctxa } = require('../utils/q'); -const { getPhotos } = require('./gamma'); - -function scrape(html) { - const { document } = ex(html); - - return ctxa(document, '.sceneList .sceneContainer').map(({ q, qa }) => { - const release = {}; - - const linkEl = q('a.imgLink'); - - release.entryId = linkEl.dataset.id; - release.title = linkEl.title; - release.url = `https://www.puretaboo.com${linkEl.href}`; - release.actors = qa('.sceneActors a', true); - - release.poster = q('.imgLink img').dataset.original; - release.trailer = `https://videothumb.gammacdn.com/307x224/${release.entryId}.mp4`; - - return release; - }); -} - -async function scrapeScene(html, url, site) { - const release = {}; - - const { q, qa, qd, qm } = ex(html); - const { pathname } = new URL(url); - - const dataPrefix = 'dataLayer = ['; - const dataStart = html.indexOf(dataPrefix); - const dataString = html.slice(dataStart + dataPrefix.length, html.indexOf('];', dataStart)); - const { sceneDetails: data } = JSON.parse(dataString); - - const dataString2 = q('script[type="application/ld+json"]', true); - const data2 = JSON.parse(dataString2)[1]; - - const videoPrefix = 'window.ScenePlayerOptions = '; - const videoStart = html.indexOf(videoPrefix); - const videoString = html.slice(videoStart + videoPrefix.length, html.indexOf('};', videoStart) + 1); - const videoData = JSON.parse(videoString); - - release.entryId = data.sceneId || videoData.playerOptions.sceneInfos.sceneId || pathname.slice(pathname.lastIndexOf('/') + 1); - release.url = url; - release.title = data.sceneTitle || videoData.playerOptions.sceneInfos.sceneTitle || qm('meta[name="twitter:title"]') || q('.sceneTitle', true); - release.description = data.sceneDescription || data2.description || q('.sceneDesc', true).replace('Video Description:', ''); - - release.duration = moment.duration(data2.duration.slice(2)).asSeconds(); - - // data2.dateCreated is the date the scene was first added to 'upcoming' - release.date = videoData.playerOptions.sceneInfos.sceneReleaseDate - ? moment.utc(videoData.playerOptions.sceneInfos.sceneReleaseDate, 'YYYY-MM-DD').toDate() - : qd('.updatedDate', 'MM-DD-YYYY'); - - release.actors = data.sceneActors?.map(actor => actor.actorName) - || data2.actor?.map(actor => actor.name) - || qa('.sceneColActors a', true); - - release.tags = data2.keywords.split(', ') || qa('.sceneColCategories a', 'title'); - - release.poster = videoData.picPreview; - release.trailer = { - src: `${videoData.playerOptions.host}${videoData.url}`, - }; - - release.photos = await getPhotos(q('.picturesItem a').href, site); - - return release; -} - -async function fetchLatest(site, page = 1) { - const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`); - - if (res.statusCode === 200) { - return scrape(res.body.toString(), site); - } - - return null; -} - -async function fetchUpcoming(site) { - const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/1/upcoming`); - - return scrape(res.body.toString(), site); -} - -async function fetchScene(url, site) { - const res = await bhttp.get(url); - - return scrapeScene(res.body.toString(), url, site); -} module.exports = { - fetchLatest, + fetchLatest: fetchApiLatest, fetchScene, - fetchUpcoming, + fetchUpcoming: fetchApiUpcoming, }; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index 4db485cf..b70e1b32 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -1,54 +1,51 @@ 'use strict'; -// releases const adulttime = require('./adulttime'); const babes = require('./babes'); const bang = require('./bang'); -const dogfart = require('./dogfart'); -const digitalplayground = require('./digitalplayground'); -const fakehub = require('./fakehub'); -const jayrock = require('./jayrock'); -const kink = require('./kink'); -const mikeadriano = require('./mikeadriano'); -const milehighmedia = require('./milehighmedia'); -const perfectgonzo = require('./perfectgonzo'); -const pervcity = require('./pervcity'); -const pornpros = require('./pornpros'); -const privateNetwork = require('./private'); // reserved keyword -const puretaboo = require('./puretaboo'); -const mindgeek = require('./mindgeek'); -const realitykings = require('./realitykings'); -const teamskeet = require('./teamskeet'); -const vixen = require('./vixen'); -const vogov = require('./vogov'); - -// releases and profiles const bangbros = require('./bangbros'); const blowpass = require('./blowpass'); +const boobpedia = require('./boobpedia'); const brazzers = require('./brazzers'); const ddfnetwork = require('./ddfnetwork'); -const famedigital = require('./famedigital'); +const digitalplayground = require('./digitalplayground'); +const dogfart = require('./dogfart'); const evilangel = require('./evilangel'); -const julesjordan = require('./julesjordan'); -const kellymadison = require('./kellymadison'); -const legalporno = require('./legalporno'); -const men = require('./men'); -const metrohd = require('./metrohd'); -const mofos = require('./mofos'); -const naughtyamerica = require('./naughtyamerica'); -const score = require('./score'); -const twentyonesextury = require('./21sextury'); -const twistys = require('./twistys'); -const xempire = require('./xempire'); -const wicked = require('./wicked'); - -// profiles -const boobpedia = require('./boobpedia'); +const fakehub = require('./fakehub'); +const famedigital = require('./famedigital'); +const fantasymassage = require('./fantasymassage'); const freeones = require('./freeones'); const freeonesLegacy = require('./freeones_legacy'); const iconmale = require('./iconmale'); -const transangels = require('./transangels'); +const jayrock = require('./jayrock'); +const julesjordan = require('./julesjordan'); +const kellymadison = require('./kellymadison'); +const kink = require('./kink'); +const legalporno = require('./legalporno'); +const men = require('./men'); +const metrohd = require('./metrohd'); +const mikeadriano = require('./mikeadriano'); +const milehighmedia = require('./milehighmedia'); +const mindgeek = require('./mindgeek'); +const mofos = require('./mofos'); +const girlsway = require('./girlsway'); +const naughtyamerica = require('./naughtyamerica'); +const perfectgonzo = require('./perfectgonzo'); +const pervcity = require('./pervcity'); const pornhub = require('./pornhub'); +const pornpros = require('./pornpros'); +const privateNetwork = require('./private'); // reserved keyword +const puretaboo = require('./puretaboo'); +const realitykings = require('./realitykings'); +const score = require('./score'); +const teamskeet = require('./teamskeet'); +const transangels = require('./transangels'); +const twentyonesextury = require('./21sextury'); +const twistys = require('./twistys'); +const vixen = require('./vixen'); +const vogov = require('./vogov'); +const wicked = require('./wicked'); +const xempire = require('./xempire'); module.exports = { releases: { @@ -67,6 +64,8 @@ module.exports = { famedigital, evilangel, fakehub, + fantasymassage, + girlsway, jayrock, julesjordan, kellymadison, diff --git a/src/sites.js b/src/sites.js index 44575788..236506fb 100644 --- a/src/sites.js +++ b/src/sites.js @@ -2,7 +2,7 @@ const config = require('config'); -const logger = require('./logger'); +const logger = require('./logger')(__filename); const argv = require('./argv'); const knex = require('./knex'); const whereOr = require('./utils/where-or'); diff --git a/src/tags.js b/src/tags.js index 494b0eb1..40e729cd 100644 --- a/src/tags.js +++ b/src/tags.js @@ -35,9 +35,11 @@ function curateTags(tags) { } async function matchTags(rawTags) { - const tags = rawTags - .concat(rawTags.map(tag => tag.toLowerCase())) - .concat(rawTags.map(tag => tag.toUpperCase())); + const filteredTags = rawTags.filter(Boolean); + + const tags = filteredTags + .concat(filteredTags.map(tag => tag.toLowerCase())) + .concat(filteredTags.map(tag => tag.toUpperCase())); const tagEntries = await knex('tags') .pluck('aliases.id') diff --git a/src/utils/upsert.js b/src/utils/upsert.js index 50c4f6e5..72a07ab1 100644 --- a/src/utils/upsert.js +++ b/src/utils/upsert.js @@ -1,7 +1,7 @@ 'use strict'; const knex = require('../knex'); -const logger = require('./logger')(__filename); +const logger = require('../logger')(__filename); async function upsert(table, items, identifier = ['id'], _knex) { const identifiers = Array.isArray(identifier) ? identifier : [identifier];