diff --git a/assets/components/tile/release.vue b/assets/components/tile/release.vue index 1b392114..97390899 100644 --- a/assets/components/tile/release.vue +++ b/assets/components/tile/release.vue @@ -48,7 +48,7 @@ Promise.resolve() table.string('url', 1000); table.string('title'); + table.string('slug'); table.date('date'); table.text('description'); diff --git a/public/img/logos/babes/babes.png b/public/img/logos/babes/babes.png index 58fce0c3..f1af5963 100644 Binary files a/public/img/logos/babes/babes.png and b/public/img/logos/babes/babes.png differ diff --git a/public/img/logos/babes/misc/babes.png b/public/img/logos/babes/misc/babes.png new file mode 100644 index 00000000..cd88c376 Binary files /dev/null and b/public/img/logos/babes/misc/babes.png differ diff --git a/public/img/logos/babes/misc/babes_soft.png b/public/img/logos/babes/misc/babes_soft.png new file mode 100644 index 00000000..f1af5963 Binary files /dev/null and b/public/img/logos/babes/misc/babes_soft.png differ diff --git a/public/img/logos/babes/misc/babes_splash.png b/public/img/logos/babes/misc/babes_splash.png new file mode 100644 index 00000000..5425781b Binary files /dev/null and b/public/img/logos/babes/misc/babes_splash.png differ diff --git a/public/img/logos/babes/network.png b/public/img/logos/babes/network.png index 58fce0c3..f1af5963 100644 Binary files a/public/img/logos/babes/network.png and b/public/img/logos/babes/network.png differ diff --git a/public/img/logos/digitalplayground/digitalplayground.png b/public/img/logos/digitalplayground/digitalplayground.png new file mode 100644 index 00000000..40641763 Binary files /dev/null and b/public/img/logos/digitalplayground/digitalplayground.png differ diff --git a/public/img/logos/evilangel/evilangel.png b/public/img/logos/evilangel/evilangel.png index 15377252..be03f1a2 100644 Binary files a/public/img/logos/evilangel/evilangel.png and b/public/img/logos/evilangel/evilangel.png differ diff --git a/public/img/logos/evilangel/misc/evil-angel.png b/public/img/logos/evilangel/misc/evil-angel.png new file mode 100644 index 00000000..cffa46bc Binary files /dev/null and b/public/img/logos/evilangel/misc/evil-angel.png differ diff --git a/public/img/logos/evilangel/misc/evil-angel.svg b/public/img/logos/evilangel/misc/evil-angel.svg new file mode 100644 index 00000000..f648cb28 --- /dev/null +++ b/public/img/logos/evilangel/misc/evil-angel.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/evilangel/network.png b/public/img/logos/evilangel/network.png index 79847560..2734e685 100644 Binary files a/public/img/logos/evilangel/network.png and b/public/img/logos/evilangel/network.png differ diff --git a/public/img/logos/kellymadison/kellymadison.png b/public/img/logos/kellymadison/kellymadison.png index ff73dde3..7781d5f4 100644 Binary files a/public/img/logos/kellymadison/kellymadison.png and b/public/img/logos/kellymadison/kellymadison.png differ diff --git a/public/img/logos/kellymadison/network.png b/public/img/logos/kellymadison/network.png index 74217ad5..9a3e1e7d 100644 Binary files a/public/img/logos/kellymadison/network.png and b/public/img/logos/kellymadison/network.png differ diff --git a/public/img/logos/kellymadison/pornfidelity.png b/public/img/logos/kellymadison/pornfidelity.png index cddb25fa..b8b8667b 100644 Binary files a/public/img/logos/kellymadison/pornfidelity.png and b/public/img/logos/kellymadison/pornfidelity.png differ diff --git a/public/img/logos/kellymadison/teenfidelity.png b/public/img/logos/kellymadison/teenfidelity.png index 44f8b53b..8b8d8058 100644 Binary files a/public/img/logos/kellymadison/teenfidelity.png and b/public/img/logos/kellymadison/teenfidelity.png differ diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 6af3ca3e..7a6aa991 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -1007,7 +1007,15 @@ const sites = [ description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars', network: 'ddfnetwork', }, - // FAKE HUB + // DIGITAL PLAYGROUND + { + slug: 'digitalplayground', + name: 'Digital Playground', + url: 'https://www.digitalplayground.com/scenes', + description: '', + parameters: { extract: true }, + network: 'digitalplayground', + }, { slug: 'episodes', name: 'Episodes', diff --git a/src/actors.js b/src/actors.js index bcd75a1f..ee8dd826 100644 --- a/src/actors.js +++ b/src/actors.js @@ -367,6 +367,7 @@ async function scrapeActors(actorNames) { } catch (error) { if (error.warn !== false) { logger.warn(`Error in scraper ${source}: ${error.message}`); + logger.error(error.stack); } } diff --git a/src/argv.js b/src/argv.js index ae608543..9e26c93f 100644 --- a/src/argv.js +++ b/src/argv.js @@ -39,7 +39,7 @@ const { argv } = yargs describe: 'Scrape profiles for new actors after fetching scenes', type: 'boolean', alias: 'with-actors', - default: true, + default: false, }) .option('scene', { describe: 'Scrape scene info from URL', diff --git a/src/releases.js b/src/releases.js index bbe14033..aa01de48 100644 --- a/src/releases.js +++ b/src/releases.js @@ -1,5 +1,6 @@ 'use strict'; +const config = require('config'); const Promise = require('bluebird'); const moment = require('moment'); @@ -16,6 +17,7 @@ const { storeTrailer, } = require('./media'); const { fetchSites, findSiteByUrl } = require('./sites'); +const slugify = require('./utils/slugify'); function commonQuery(queryBuilder, { filter = [], @@ -204,6 +206,11 @@ async function attachStudio(release) { } async function curateReleaseEntry(release) { + const slug = slugify(release.title, { + encode: true, + limit: config.titleSlugLength, + }); + const curatedRelease = { site_id: release.site.id, studio_id: release.studio ? release.studio.id : null, @@ -213,6 +220,7 @@ async function curateReleaseEntry(release) { type: release.type, url: release.url, title: release.title, + slug, date: release.date, description: release.description, // director: release.director, @@ -397,7 +405,7 @@ async function storeRelease(release) { logger.info(`Stored release "${release.title}" (${releaseEntry.id}, ${release.site.name})`); - return releaseEntry.id; + return releaseEntry; } async function storeReleases(releases) { @@ -405,10 +413,11 @@ async function storeReleases(releases) { try { const releaseWithChannelSite = await attachChannelSite(release); const releaseWithStudio = await attachStudio(releaseWithChannelSite); - const releaseId = await storeRelease(releaseWithStudio); + const { id, slug } = await storeRelease(releaseWithStudio); return { - id: releaseId, + id, + slug, ...releaseWithChannelSite, }; } catch (error) { diff --git a/src/scrape-releases.js b/src/scrape-releases.js index 96545168..ed7f7722 100644 --- a/src/scrape-releases.js +++ b/src/scrape-releases.js @@ -97,7 +97,7 @@ async function scrapeReleases(sources, release = null, type = 'scene') { const { releases: storedReleases } = await storeReleases(curatedReleases); if (storedReleases) { - console.log(storedReleases.map(storedRelease => `http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`).join('\n')); + logger.info(storedReleases.map(storedRelease => `\nhttp://${config.web.host}:${config.web.port}/scene/${storedRelease.id}/${storedRelease.slug}`).join('')); } } } diff --git a/src/scrapers/jayrock.js b/src/scrapers/jayrock.js index 3a82e1a6..88d6816a 100644 --- a/src/scrapers/jayrock.js +++ b/src/scrapers/jayrock.js @@ -58,7 +58,7 @@ async function scrapeScene(scene, site, tokens) { }, }; - release.url = `${site.url}/scene/${release.entryId}/${slugify(release.title, true)}`; + release.url = `${site.url}/scene/${release.entryId}/${slugify(release.title, { encode: true })}`; release.date = new Date(scene.sites.collection[scene.id].publishDate); release.poster = scene._resources.primary[0].url; diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index a6649947..4c40a11d 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -26,6 +26,16 @@ function getThumbs(scene) { } function scrapeLatestX(data, site) { + if (site.parameters?.extract === true && data.collections.length > 0) { + // release should not belong to any channel + return null; + } + + if (typeof site.parameters?.extract === 'string' && !data.collections.some(collection => collection.shortName === site.parameters.extract)) { + // release should belong to specific channel + return null; + } + const { id: entryId, title, description } = data; const hostname = site.parameters?.native ? site.url : site.network.url; const url = `${hostname}/scene/${entryId}/`; @@ -58,7 +68,9 @@ function scrapeLatestX(data, site) { } async function scrapeLatest(items, site) { - return Promise.all(items.map(async data => scrapeLatestX(data, site))); + const latestReleases = await Promise.all(items.map(async data => scrapeLatestX(data, site))); + + return latestReleases.filter(Boolean); } function scrapeScene(data, url, _site) { @@ -85,10 +97,10 @@ function scrapeScene(data, url, _site) { }; } - const siteName = data.collections[0].name; + const siteName = data.collections[0]?.name || data.brand; release.channel = siteName.replace(/\s+/g, '').toLowerCase(); - release.url = url || `https://www.realitykings.com/scene/${entryId}/`; + release.url = url || `https://www.${data.brand}.com/scene/${entryId}/`; return release; } @@ -104,6 +116,9 @@ function getUrl(site) { return `${site.url}/scenes`; } + if (site.parameters?.extract) { + return `${site.url}/scenes`; + } if (site.parameters?.siteId) { return `${site.network.url}/scenes?site=${site.parameters.siteId}`; @@ -144,7 +159,7 @@ function scrapeProfile(data, html, releases = []) { if (data.height) profile.height = inchesToCm(data.height); if (data.weight) profile.weight = lbsToKg(data.weight); - if (data.images.card_main_rect && data.images.card_main_rect[0]) { + if (data.images.card_main_rect?.[0]) { profile.avatar = data.images.card_main_rect[0].xl?.url || data.images.card_main_rect[0].lg?.url || data.images.card_main_rect[0].md?.url @@ -169,7 +184,7 @@ async function fetchLatest(site, page = 1) { const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const limit = 10; - const apiUrl = site.parameters?.native + const apiUrl = site.parameters?.native || site.parameters?.extract ? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene` : `https://site-api.project1service.com/v2/releases?collectionId=${siteId}&dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`; diff --git a/src/scrapers/score.js b/src/scrapers/score.js index b21b93b4..f8ab3482 100644 --- a/src/scrapers/score.js +++ b/src/scrapers/score.js @@ -77,7 +77,7 @@ async function scrapeScene(html, url, site) { release.actors = qa('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true); if (release.actors.length === 0) { - const actorEl = qa('.stat').find(stat => /Featuring/.test(stat.textContent)) + const actorEl = qa('.stat').find(stat => /Featuring/.test(stat.textContent)); const actorString = qtext(actorEl); console.log(actorString); @@ -147,7 +147,7 @@ function scrapeProfile(html) { const bio = qa('.stat').reduce((acc, el) => { const prop = q(el, '.label', true).slice(0, -1); - const key = slugify(prop, false, '_'); + const key = slugify(prop, { delimiter: '_' }); const value = q(el, '.value', true); return { diff --git a/src/sites.js b/src/sites.js index 4a2ed0a4..f85b5bfa 100644 --- a/src/sites.js +++ b/src/sites.js @@ -60,7 +60,7 @@ function destructConfigNetworks(networks = []) { } async function findSiteByUrl(url) { - const { origin, pathname } = new URL(url); + const { origin, hostname, pathname } = new URL(url); // const domain = hostname.replace(/www.|tour./, ''); const dirUrl = `${origin}${pathname.split('/').slice(0, 2).join('/')}`; // allow for sites on URI directory @@ -72,6 +72,9 @@ async function findSiteByUrl(url) { ) .where('sites.url', url) .orWhere('sites.url', origin) + .orWhere('sites.url', origin.replace(/www\.|tour\./, '')) + .orWhere('sites.url', `https://www.${hostname}`) + .orWhere('sites.url', `http://www.${hostname}`) .orWhere('sites.url', dirUrl) // .orWhere('sites.url', 'like', `%${domain}`) .first(); diff --git a/src/utils/posters.js b/src/utils/posters.js index 4aab63ee..eefe9508 100644 --- a/src/utils/posters.js +++ b/src/utils/posters.js @@ -9,16 +9,18 @@ const knex = require('../knex'); async function init() { const posters = await knex('actors') - .select('actors.name', 'releases.title', 'media.path') - .whereIn('name', argv.actors) + .select('actors.name as actor_name', 'releases.title', 'media.path', 'sites.name as site_name', 'networks.name as network_name') + .whereIn('actors.name', argv.actors) .join('releases_actors', 'releases_actors.actor_id', 'actors.id') .join('releases', 'releases_actors.release_id', 'releases.id') .join('releases_posters', 'releases_posters.release_id', 'releases.id') + .join('sites', 'sites.id', 'releases.site_id') + .join('networks', 'networks.id', 'sites.network_id') .join('media', 'releases_posters.media_id', 'media.id'); await Promise.all(posters.map(async (poster) => { const source = path.join(config.media.path, poster.path); - const target = path.join(config.media.path, 'posters', `${poster.title.replace('/', '_')}.${poster.name}.jpeg`); + const target = path.join(config.media.path, 'posters', `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`); const file = await fs.readFile(source); await fs.writeFile(target, file); diff --git a/src/utils/slugify.js b/src/utils/slugify.js index 292f8686..9e7c2ff3 100644 --- a/src/utils/slugify.js +++ b/src/utils/slugify.js @@ -1,7 +1,21 @@ 'use strict'; -function slugify(string, encode = false, delimiter = '-') { - const slug = string.trim().toLowerCase().match(/\w+/g).join(delimiter); +function slugify(string, { + encode = false, + delimiter = '-', + limit = 1000, +} = {}) { + const slugComponents = string.trim().toLowerCase().match(/\w+/g); + + const slug = slugComponents.reduce((acc, component, index) => { + const accSlug = `${acc}${index > 0 ? delimiter : ''}${component}`; + + if (accSlug.length < limit) { + return accSlug; + } + + return acc; + }, ''); return encode ? encodeURI(slug) : slug; }