diff --git a/assets/components/actors/actors.vue b/assets/components/actors/actors.vue index 6e33ecaf..f38894b2 100644 --- a/assets/components/actors/actors.vue +++ b/assets/components/actors/actors.vue @@ -85,7 +85,7 @@ export default { actors: [], pageTitle: null, letters: ['all'].concat(Array.from({ length: 26 }, (value, index) => String.fromCharCode(index + 97).toUpperCase())), - letter: this.$route.params.letter, + letter: this.$route.params.letter || 'all', gender: this.$route.params.gender || 'female', }; }, diff --git a/assets/js/router.js b/assets/js/router.js index a7b40e25..4db6fd13 100644 --- a/assets/js/router.js +++ b/assets/js/router.js @@ -51,18 +51,7 @@ const routes = [ name: 'tag', }, { - path: '/actors', - component: Actors, - redirect: { - name: 'actors', - params: { - gender: 'female', - letter: 'all', - }, - }, - }, - { - path: '/actors/:gender/:letter', + path: '/actors/:gender?/:letter?', component: Actors, name: 'actors', }, diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index fba43bb6..7a8e07c1 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -105,7 +105,7 @@ exports.up = knex => Promise.resolve() table.string('name'); table.string('url'); table.text('description'); - table.text('parameters'); + table.json('parameters'); table.string('slug', 32) .unique(); @@ -140,7 +140,7 @@ exports.up = knex => Promise.resolve() table.string('name'); table.string('url'); table.text('description'); - table.text('parameters'); + table.json('parameters'); table.integer('priority', 3) .defaultTo(0); diff --git a/public/img/logos/jayrock/blackforwife.png b/public/img/logos/jayrock/blackforwife.png new file mode 100644 index 00000000..77b9ad67 Binary files /dev/null and b/public/img/logos/jayrock/blackforwife.png differ diff --git a/public/img/logos/jayrock/misc/black-for-wife.png b/public/img/logos/jayrock/misc/black-for-wife.png new file mode 100644 index 00000000..9929c8b6 Binary files /dev/null and b/public/img/logos/jayrock/misc/black-for-wife.png differ diff --git a/public/img/logos/jayrock/misc/black-for-wife.svg b/public/img/logos/jayrock/misc/black-for-wife.svg new file mode 100644 index 00000000..bb63b476 --- /dev/null +++ b/public/img/logos/jayrock/misc/black-for-wife.svg @@ -0,0 +1,108 @@ + + + + + + + + + diff --git a/seeds/00_networks.js b/seeds/00_networks.js index 23303d9f..6d715702 100644 --- a/seeds/00_networks.js +++ b/seeds/00_networks.js @@ -83,7 +83,6 @@ const networks = [ name: 'Dogfart Network', url: 'https://dogfartnetwork.com', description: 'The world famous Dogfart Interracial series. Online since 1996, we have the largest collection of Interracial videos, pictures and content on the web.', - parameters: JSON.stringify({ photoLimit: 25 }), }, { slug: 'evilangel', diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 14a38e1e..5d0589bc 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -1673,6 +1673,18 @@ const sites = [ url: 'https://cospimps.com', network: 'jayrock', }, + { + slug: 'blackforwife', + name: 'Black for Wife', + url: 'https://www.blackforwife.com', + network: 'jayrock', + parameters: { + referer: 'https://freetour.adulttime.com/en/blackforwife', + useGamma: true, + scene: false, + deep: 'https://21sextury.com/en/video/', + }, + }, // JULES JORDAN { slug: 'julesjordan', @@ -4447,7 +4459,7 @@ exports.seed = knex => Promise.resolve() name: site.name, description: site.description, url: site.url, - parameters: JSON.stringify(site.parameters), + parameters: site.parameters, network_id: networksMap[site.network], priority: site.priority, scrape: site.scrape, diff --git a/src/actors.js b/src/actors.js index 8fc00bcf..8bd677ba 100644 --- a/src/actors.js +++ b/src/actors.js @@ -15,10 +15,15 @@ const slugify = require('./utils/slugify'); const { createMediaDirectory, storePhotos } = require('./media'); async function curateActor(actor) { - const [aliases, photos, social] = await Promise.all([ + const [aliases, avatar, photos, social] = await Promise.all([ knex('actors').where({ alias_for: actor.id }), - knex('media') - .where({ domain: 'actors', target_id: actor.id }) + knex('actors_avatars') + .where('actor_id', actor.id) + .join('media', 'media.id', 'actors_avatars.media_id') + .first(), + knex('actors_photos') + .where('actor_id', actor.id) + .join('media', 'media.id', 'actors_photos.media_id') .orderBy('index'), knex('actors_social') .where('actor_id', actor.id) @@ -43,8 +48,8 @@ async function curateActor(actor) { naturalBoobs: actor.natural_boobs, aliases: aliases.map(({ name }) => name), slug: actor.slug, - avatar: photos.find(photo => photo.role === 'avatar'), - photos: photos.filter(photo => photo.role === 'photo'), + avatar, + photos, hasTattoos: actor.has_tattoos, hasPiercings: actor.has_piercings, tattoos: actor.tattoos, diff --git a/src/networks.js b/src/networks.js index ab1da0d5..d0fdd643 100644 --- a/src/networks.js +++ b/src/networks.js @@ -18,7 +18,7 @@ async function curateNetwork(network, includeParameters = false) { description: network.description, slug: network.slug, sites, - parameters: includeParameters ? JSON.parse(network.parameters) : null, + parameters: includeParameters ? network.parameters : null, studios: studios.map(studio => ({ id: studio.id, name: studio.name, diff --git a/src/releases.js b/src/releases.js index a55b3488..40a6527c 100644 --- a/src/releases.js +++ b/src/releases.js @@ -125,9 +125,7 @@ async function curateRelease(release) { site: { id: release.site_id, name: release.site_name, - independent: release.site_parameters - ? (JSON.parse(release.site_parameters).independent || false) - : false, + independent: !!release.site_parameters?.independent, slug: release.site_slug, url: release.site_url, }, diff --git a/src/scrape-releases.js b/src/scrape-releases.js index 4f0712b0..43e07451 100644 --- a/src/scrape-releases.js +++ b/src/scrape-releases.js @@ -65,7 +65,6 @@ async function scrapeRelease(source, basicRelease = null, type = 'scene') { : await scraper.fetchMovie(url, site, release); return { - url, ...release, ...scrapedRelease, ...(scrapedRelease && release?.tags && { diff --git a/src/scrape-sites.js b/src/scrape-sites.js index 2aa3ee4b..3a5eb9b2 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -68,11 +68,13 @@ async function scrapeUniqueReleases(scraper, site, afterDate = getAfterDate(), a return scrapeUniqueReleases(scraper, site, afterDate, accReleases.concat(uniqueReleases), page + 1); } + const uniqueReleasesWithSite = uniqueReleases.map(release => ({ ...release, site })); + if (oldestReleaseOnPage) { - return accReleases.concat(uniqueReleases); + return accReleases.concat(uniqueReleasesWithSite); } - return accReleases.concat(uniqueReleases).slice(0, argv.nullDateLimit); + return accReleases.concat(uniqueReleasesWithSite).slice(0, argv.nullDateLimit); } async function scrapeUpcomingReleases(scraper, site) { @@ -87,9 +89,9 @@ async function scrapeUpcomingReleases(scraper, site) { async function deepFetchReleases(baseReleases) { return Promise.map(baseReleases, async (release) => { - if (release.url) { + if (release.url || (release.path && release.site)) { try { - const fullRelease = await scrapeRelease(release.url, release, 'scene'); + const fullRelease = await scrapeRelease(release.url || release.path, release, 'scene'); if (fullRelease) { return { diff --git a/src/scrapers/gamma.js b/src/scrapers/gamma.js index 4174ddd1..05862ce4 100644 --- a/src/scrapers/gamma.js +++ b/src/scrapers/gamma.js @@ -100,13 +100,14 @@ async function scrapeApiReleases(json, site) { dislikes: scene.ratings_down, }; - release.url = site.parameters?.scene - ? `${site.parameters.scene}/${scene.url_title}/${release.entryId}` - : `${site.url}/en/video/${scene.url_title}/${release.entryId}`; + release.path = `/${scene.url_title}/${release.entryId}`; + + if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`; + else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`; release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate(); release.actors = scene.actors.map(({ name }) => name); - release.director = scene.directors[0].name; + release.director = scene.directors[0]?.name || null; release.tags = scene.master_categories .concat(scene.categories?.map(category => category.name)) @@ -171,9 +172,9 @@ function scrapeAll(html, site, networkUrl, hasTeaser = true) { }); } -async function scrapeScene(html, url, site) { +async function scrapeScene(html, url, site, scrapedRelease) { const $ = cheerio.load(html, { normalizeWhitespace: true }); - const release = { $, url }; + const release = { $ }; const json = $('script[type="application/ld+json"]').html(); const videoJson = $('script:contains("window.ScenePlayerOptions")').html(); @@ -181,7 +182,7 @@ async function scrapeScene(html, url, site) { const [data, data2] = json ? JSON.parse(json) : []; const videoData = JSON.parse(videoJson.slice(videoJson.indexOf('{'), videoJson.indexOf('};') + 1)); - [release.entryId] = new URL(url).pathname.split('/').slice(-1); + [release.entryId] = (scrapedRelease?.path || new URL(url).pathname).split('/').slice(-1); release.title = videoData?.playerOptions?.sceneInfos.sceneTitle || data?.name; // date in data object is not the release date of the scene, but the date the entry was added; only use as fallback @@ -194,7 +195,8 @@ async function scrapeScene(html, url, site) { if (data) { release.description = data.description; - release.director = data.director?.[0].name || data2?.director?.[0].name; + if (data.director?.[0]?.name) release.director = data.director[0].name; + else if (data2?.director?.[0]?.name) release.director = data2.director[0].name; const actors = data?.actor || data2?.actor || []; release.actors = actors.map(actor => actor.name); @@ -437,8 +439,8 @@ async function fetchUpcoming(site) { return scrapeAll(res.body.toString(), site); } -function getDeepUrl(url, site) { - const { pathname } = new URL(url); +function getDeepUrl(url, site, release) { + const pathname = release.path || new URL(url).pathname; if (site.parameters?.deep === 'network') { return `${site.network.url}${pathname}`; @@ -456,11 +458,11 @@ async function fetchScene(url, site, release) { return release; } - const deepUrl = getDeepUrl(url, site); + const deepUrl = getDeepUrl(url, site, release); const res = await bhttp.get(deepUrl); if (res.statusCode === 200) { - return scrapeScene(res.body.toString(), url, site, deepUrl); + return scrapeScene(res.body.toString(), url, site, release); } return null; diff --git a/src/scrapers/jayrock.js b/src/scrapers/jayrock.js index d481acc9..d3dba53b 100644 --- a/src/scrapers/jayrock.js +++ b/src/scrapers/jayrock.js @@ -6,6 +6,7 @@ const bhttp = require('bhttp'); const logger = require('../logger')(__filename); const slugify = require('../utils/slugify'); +const { fetchApiLatest, fetchScene } = require('./gamma'); async function fetchToken(site) { const res = await bhttp.get(site.url); @@ -81,6 +82,10 @@ function scrapeLatest(scenes, site, tokens) { } async function fetchLatest(site, page = 1) { + if (site.parameters?.useGamma) { + return fetchApiLatest(site, page); + } + const { time, token } = await fetchToken(site); // transParameters[v1] includes _resources, [v2] includes photos, [preset] is mandatory @@ -94,7 +99,11 @@ async function fetchLatest(site, page = 1) { return null; } -async function fetchScene(url, site, release) { +async function fetchNetworkScene(url, site, release) { + if (site.parameters?.useGamma) { + return fetchScene(url, site, release); + } + const { time, token } = release?.meta.tokens || await fetchToken(site); // use attached tokens when deep fetching const { pathname } = new URL(url); const entryId = pathname.split('/')[2]; @@ -111,5 +120,5 @@ async function fetchScene(url, site, release) { module.exports = { fetchLatest, - fetchScene, + fetchScene: fetchNetworkScene, }; diff --git a/src/sites.js b/src/sites.js index 236506fb..b7632d4a 100644 --- a/src/sites.js +++ b/src/sites.js @@ -13,8 +13,6 @@ async function curateSite(site, includeParameters = false) { .where('site_id', site.id) .join('tags', 'tags.id', 'sites_tags.tag_id'); - const parameters = JSON.parse(site.parameters); - return { id: site.id, name: site.name, @@ -22,15 +20,15 @@ async function curateSite(site, includeParameters = false) { description: site.description, slug: site.slug, tags, - independent: !!parameters && parameters.independent, - parameters: includeParameters ? parameters : null, + independent: !!site.parameters && site.parameters.independent, + parameters: includeParameters ? site.parameters : null, network: { id: site.network_id, name: site.network_name, description: site.network_description, slug: site.network_slug, url: site.network_url, - parameters: includeParameters ? JSON.parse(site.network_parameters) : null, + parameters: includeParameters ? site.network_parameters : null, }, }; } diff --git a/src/utils/list.js b/src/utils/list.js index f96a1f11..d847cef1 100644 --- a/src/utils/list.js +++ b/src/utils/list.js @@ -23,7 +23,7 @@ async function listSites() { sites.forEach((site) => { const rkSpecial = network.id === 'realitykings' && (new URL(site.url).hostname === 'www.realitykings.com' - || (site.parameters && JSON.parse(site.parameters).altLayout)) + || (site.parameters?.altLayout)) ? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL console.log(` * ${site.name}${rkSpecial}`); diff --git a/src/web/plugins/sites.js b/src/web/plugins/sites.js index 44dbdb02..5e25dfee 100644 --- a/src/web/plugins/sites.js +++ b/src/web/plugins/sites.js @@ -11,13 +11,7 @@ const schemaExtender = makeExtendSchemaPlugin(_build => ({ resolvers: { Site: { independent(parent, _args, _context, _info) { - if (!parent.parameters) { - return false; - } - - const parameters = JSON.parse(parent.parameters); - - return !!parameters.independent; + return !!parent.parameters?.independent; }, }, },