diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index b3cea235..f5ad458e 100644 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -208,11 +208,24 @@ class="link added" >{{ formatDate(release.dateAdded, 'MMMM D, YYYY') }} + +
+ + + +
@@ -375,6 +403,12 @@ export default { margin: 0 1rem .5rem 0; } +.filename { + width: 100%; + padding: .5rem; + border: solid 1px $shadow-weak; +} + .link { display: inline-block; color: $link; diff --git a/assets/components/tile/site.vue b/assets/components/tile/site.vue index 1a9b75fe..65ba2f55 100644 --- a/assets/components/tile/site.vue +++ b/assets/components/tile/site.vue @@ -46,7 +46,7 @@ export default { .logo { width: 100%; height: 5rem; - color: $text; + color: $text-contrast; display: flex; align-items: center; justify-content: center; diff --git a/assets/img/file-empty2.svg b/assets/img/file-empty2.svg new file mode 100644 index 00000000..ddc98055 --- /dev/null +++ b/assets/img/file-empty2.svg @@ -0,0 +1,6 @@ + + +file-empty2 + + + diff --git a/assets/img/file-video2.svg b/assets/img/file-video2.svg new file mode 100644 index 00000000..482f591d --- /dev/null +++ b/assets/img/file-video2.svg @@ -0,0 +1,6 @@ + + +file-video2 + + + diff --git a/assets/img/paste2.svg b/assets/img/paste2.svg new file mode 100644 index 00000000..9cea16df --- /dev/null +++ b/assets/img/paste2.svg @@ -0,0 +1,5 @@ + + +paste2 + + diff --git a/assets/img/paste3.svg b/assets/img/paste3.svg new file mode 100644 index 00000000..e0ccb3fe --- /dev/null +++ b/assets/img/paste3.svg @@ -0,0 +1,6 @@ + + +paste3 + + + diff --git a/assets/img/unlocked.svg b/assets/img/unlocked.svg new file mode 100644 index 00000000..ae51d01f --- /dev/null +++ b/assets/img/unlocked.svg @@ -0,0 +1,5 @@ + + +unlocked + + diff --git a/assets/js/config/default.js b/assets/js/config/default.js index fccd0167..3d86df36 100644 --- a/assets/js/config/default.js +++ b/assets/js/config/default.js @@ -2,4 +2,9 @@ export default { api: { url: `${window.location.origin}/api`, }, + filename: { + pattern: '{site.name} - {title} ({actors.$n.name}, {date} {shootId})', + separator: ', ', + date: 'DD-MM-YYYY', + }, }; diff --git a/assets/js/networks/actions.js b/assets/js/networks/actions.js index 7bfd452a..fb97ec76 100644 --- a/assets/js/networks/actions.js +++ b/assets/js/networks/actions.js @@ -19,7 +19,7 @@ function initNetworksActions(store, _router) { slug url sites( - orderBy: PRIORITY_DESC, + orderBy: [PRIORITY_DESC, NAME_ASC], filter: { enabled: { equalTo: true, diff --git a/package-lock.json b/package-lock.json index 98f3794c..658371d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11143,6 +11143,11 @@ "resolved": "https://registry.npmjs.org/tarn/-/tarn-1.1.5.tgz", "integrity": "sha512-PMtJ3HCLAZeedWjJPgGnCvcphbCOMbtZpjKgLq3qM5Qq9aQud+XHrL0WlrlgnTyS8U+jrjGbEXprFcQrxPy52g==" }, + "template-format": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/template-format/-/template-format-1.2.5.tgz", + "integrity": "sha512-ZZqSfqYBMfPjouADYSRN9iaYlLr2PPVFYgULcV8cGMrJbifNXKvP7qx5PBFQjXg5mh1Gwkk+LTgdsZ8bmSvBdw==" + }, "terser": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/terser/-/terser-4.4.2.tgz", diff --git a/package.json b/package.json index e014a803..25e38d0b 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "sharp": "^0.23.4", "showdown": "^1.9.1", "source-map-support": "^0.5.16", + "template-format": "^1.2.5", "tough-cookie": "^3.0.1", "tty-table": "^2.8.3", "url-pattern": "^1.0.3", diff --git a/public/img/logos/21sextury/misc/21-sextury.svg b/public/img/logos/21sextury/misc/21-sextury.svg new file mode 100644 index 00000000..3acf4226 --- /dev/null +++ b/public/img/logos/21sextury/misc/21-sextury.svg @@ -0,0 +1,2 @@ + +21sextury-logo \ No newline at end of file diff --git a/public/img/logos/21sextury/network.png b/public/img/logos/21sextury/network.png index 3b16476c..d3b217c2 100644 Binary files a/public/img/logos/21sextury/network.png and b/public/img/logos/21sextury/network.png differ diff --git a/public/img/logos/bang/bang.png b/public/img/logos/bang/bang.png deleted file mode 100644 index 122642e3..00000000 Binary files a/public/img/logos/bang/bang.png and /dev/null differ diff --git a/public/img/logos/bang/misc/bang.png b/public/img/logos/bang/misc/bang.png new file mode 100644 index 00000000..894d2d0f Binary files /dev/null and b/public/img/logos/bang/misc/bang.png differ diff --git a/public/img/logos/bang/misc/bang.svg b/public/img/logos/bang/misc/bang.svg new file mode 100755 index 00000000..3dd74189 --- /dev/null +++ b/public/img/logos/bang/misc/bang.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/logos/bang/network.png b/public/img/logos/bang/network.png index acb808b8..894d2d0f 100644 Binary files a/public/img/logos/bang/network.png and b/public/img/logos/bang/network.png differ diff --git a/public/img/logos/bangbros/misc/bang-bros.png b/public/img/logos/bangbros/misc/bang-bros.png new file mode 100644 index 00000000..b2d8f51f Binary files /dev/null and b/public/img/logos/bangbros/misc/bang-bros.png differ diff --git a/public/img/logos/bangbros/misc/bang-bros.svg b/public/img/logos/bangbros/misc/bang-bros.svg new file mode 100644 index 00000000..97933f57 --- /dev/null +++ b/public/img/logos/bangbros/misc/bang-bros.svg @@ -0,0 +1,324 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/public/img/logos/bangbros/misc/bang-bros_clean.png b/public/img/logos/bangbros/misc/bang-bros_clean.png new file mode 100644 index 00000000..48b5a316 Binary files /dev/null and b/public/img/logos/bangbros/misc/bang-bros_clean.png differ diff --git a/public/img/logos/bangbros/misc/bang-bros_clean.svg b/public/img/logos/bangbros/misc/bang-bros_clean.svg new file mode 100644 index 00000000..57c1f2cf --- /dev/null +++ b/public/img/logos/bangbros/misc/bang-bros_clean.svg @@ -0,0 +1,263 @@ + +image/svg+xml \ No newline at end of file diff --git a/public/img/logos/bangbros/network.png b/public/img/logos/bangbros/network.png index 7d8c5794..ffe6e815 100644 Binary files a/public/img/logos/bangbros/network.png and b/public/img/logos/bangbros/network.png differ diff --git a/public/img/logos/blowpass/misc/blow-pass.svg b/public/img/logos/blowpass/misc/blow-pass.svg new file mode 100644 index 00000000..3e851284 --- /dev/null +++ b/public/img/logos/blowpass/misc/blow-pass.svg @@ -0,0 +1,267 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/blowpass/misc/blow-pass_light.png b/public/img/logos/blowpass/misc/blow-pass_light.png new file mode 100644 index 00000000..2f9e4b01 Binary files /dev/null and b/public/img/logos/blowpass/misc/blow-pass_light.png differ diff --git a/public/img/logos/blowpass/misc/blow-pass_light.svg b/public/img/logos/blowpass/misc/blow-pass_light.svg new file mode 100644 index 00000000..3a0a47cf --- /dev/null +++ b/public/img/logos/blowpass/misc/blow-pass_light.svg @@ -0,0 +1,320 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/blowpass/network.png b/public/img/logos/blowpass/network.png index 5f73e98d..d38bc673 100644 Binary files a/public/img/logos/blowpass/network.png and b/public/img/logos/blowpass/network.png differ diff --git a/public/img/logos/digitalplayground/favicon.png b/public/img/logos/digitalplayground/favicon.png index 3035862b..9741de31 100644 Binary files a/public/img/logos/digitalplayground/favicon.png and b/public/img/logos/digitalplayground/favicon.png differ diff --git a/public/img/logos/digitalplayground/misc/digital-playground_original.png b/public/img/logos/digitalplayground/misc/digital-playground_original.png new file mode 100644 index 00000000..707d4609 Binary files /dev/null and b/public/img/logos/digitalplayground/misc/digital-playground_original.png differ diff --git a/public/img/logos/digitalplayground/misc/digital-playground_original.svg b/public/img/logos/digitalplayground/misc/digital-playground_original.svg new file mode 100644 index 00000000..49855583 --- /dev/null +++ b/public/img/logos/digitalplayground/misc/digital-playground_original.svg @@ -0,0 +1,107 @@ + + + Digital Playground logo + + + + image/svg+xml + + Digital Playground logo + + 14 enero 2020 + + + EEIM + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/img/logos/digitalplayground/misc/favicon_border.png b/public/img/logos/digitalplayground/misc/favicon_border.png new file mode 100644 index 00000000..9f1c6f90 Binary files /dev/null and b/public/img/logos/digitalplayground/misc/favicon_border.png differ diff --git a/public/img/logos/digitalplayground/network.png b/public/img/logos/digitalplayground/network.png index b1458fe1..40641763 100644 Binary files a/public/img/logos/digitalplayground/network.png and b/public/img/logos/digitalplayground/network.png differ diff --git a/public/img/logos/fakehub/misc/network_original_large.png b/public/img/logos/fakehub/misc/network_original_large.png new file mode 100644 index 00000000..80767e9f Binary files /dev/null and b/public/img/logos/fakehub/misc/network_original_large.png differ diff --git a/public/img/logos/fakehub/network.png b/public/img/logos/fakehub/network.png index 03aa4d5d..0b08a350 100644 Binary files a/public/img/logos/fakehub/network.png and b/public/img/logos/fakehub/network.png differ diff --git a/public/img/logos/realitykings/blackgfs.png b/public/img/logos/realitykings/blackgfs.png index 59d4f849..40719379 100644 Binary files a/public/img/logos/realitykings/blackgfs.png and b/public/img/logos/realitykings/blackgfs.png differ diff --git a/public/img/logos/teamskeet/network.png b/public/img/logos/teamskeet/network.png index b2a1b3a3..babf5af9 100644 Binary files a/public/img/logos/teamskeet/network.png and b/public/img/logos/teamskeet/network.png differ diff --git a/seeds/01_sites.js b/seeds/01_sites.js index 7e48625e..6af3ca3e 100644 --- a/seeds/01_sites.js +++ b/seeds/01_sites.js @@ -1931,6 +1931,12 @@ const sites = [ url: 'https://www.naughtyamerica.com/site/big-cock-bully', network: 'naughtyamerica', }, + { + slug: 'bigcockhero', + name: 'Big Cock Hero', + url: 'https://www.naughtyamerica.com/site/big-cock-hero', + network: 'naughtyamerica', + }, { slug: 'mysistershotfriend', name: "My Sister's Hot Friend", @@ -3081,6 +3087,7 @@ const sites = [ name: 'Ashley Sage Ellison', slug: 'ashleysageellison', url: 'https://www.bigboobbundle.com/ashleysageellison', + parameters: { path: '/videos', actors: ['Ashley Sage Ellison'] }, network: 'score', }, { @@ -3088,12 +3095,14 @@ const sites = [ slug: 'autumnjade', url: 'https://www.bigboobbundle.com/autumn-jade', network: 'score', + parameters: { path: '/videos', actors: ['Autumn Jade'] }, }, { name: 'Big Boob Bundle', slug: 'bigboobbundle', url: 'https://www.bigboobbundle.com', network: 'score', + enabled: false, // all content appears to be on subsites }, { name: 'Big Boobs POV', @@ -3105,12 +3114,14 @@ const sites = [ name: 'Big Tit Angela White', slug: 'bigtitangelawhite', url: 'https://www.bigboobbundle.com/bigtitangelawhite', + parameters: { path: '/videos', actors: ['Angela White'] }, // no dates available network: 'score', }, { name: 'Big Tit Hitomi', slug: 'bigtithitomi', url: 'https://www.bigboobbundle.com/bigtithitomi', + parameters: { path: '/videos', actors: ['Hitomi'] }, network: 'score', }, { @@ -3123,6 +3134,7 @@ const sites = [ name: 'Big Tit Terry Nova', slug: 'bigtitterrynova', url: 'https://www.bigboobbundle.com/bigtitterrynova', + parameters: { path: '/videos', actors: ['Terry Nova'] }, network: 'score', }, { diff --git a/src/actors.js b/src/actors.js index 17025fab..bcd75a1f 100644 --- a/src/actors.js +++ b/src/actors.js @@ -437,19 +437,18 @@ async function scrapeBasicActors() { } async function associateActors(mappedActors, releases) { - const actorNames = Object.keys(mappedActors); - const actorSlugs = actorNames.map(name => slugify(name)); + const actorMap = Object.keys(mappedActors).reduce((acc, actorName) => ({ ...acc, [actorName]: slugify(actorName) }), {}); const [existingActorEntries, existingAssociationEntries] = await Promise.all([ knex('actors') - .whereIn('name', actorNames) - .orWhereIn('slug', actorSlugs), + .whereIn('name', Object.keys(actorMap)) + .orWhereIn('slug', Object.values(actorMap)), knex('releases_actors').whereIn('release_id', releases.map(release => release.id)), ]); const associations = await Promise.map(Object.entries(mappedActors), async ([actorName, releaseIds]) => { try { - const actorEntry = existingActorEntries.find(actor => actor.name === actorName) + const actorEntry = existingActorEntries.find(actor => actor.slug === actorMap[actorName]) || await storeActor({ name: actorName }); return releaseIds @@ -469,7 +468,7 @@ async function associateActors(mappedActors, releases) { await knex('releases_actors').insert(associations.filter(association => association).flat()); - // basic actor scraping is failure prone, don't together with actor association + // basic actor scraping is failure prone, don't run together with actor association // await scrapebasicactors(), } diff --git a/src/scrapers/naughtyamerica.js b/src/scrapers/naughtyamerica.js index 69ecb56e..b3a50e5b 100644 --- a/src/scrapers/naughtyamerica.js +++ b/src/scrapers/naughtyamerica.js @@ -6,7 +6,7 @@ const cheerio = require('cheerio'); const moment = require('moment'); const slugify = require('../utils/slugify'); -const { ex } = require('../utils/q'); +const { ex, get } = require('../utils/q'); function titleExtractor(pathname) { const components = pathname.split('/')[2].split('-'); @@ -100,7 +100,13 @@ function scrapeScene(html, url, site) { }; } -function scrapeProfile(html) { +async function fetchActorReleases(url) { + const { qus } = await get(url); + + return qus('.contain-block:not(.live-scenes) .scene-item > a:first-child'); // live scenes repeat on all pages +} + +async function scrapeProfile(html) { const { q, qus } = ex(html); const profile = {}; @@ -109,7 +115,11 @@ function scrapeProfile(html) { const avatar = q('img.performer-pic', 'src'); if (avatar) profile.avatar = `https:${avatar}`; - profile.releases = qus('.scene-item > a:first-child'); + const releases = qus('.scene-item > a:first-child'); + const otherPages = qus('.pagination a:not([rel=next]):not([rel=prev])'); + const olderReleases = await Promise.all(otherPages.map(async page => fetchActorReleases(page))); + + profile.releases = releases.concat(olderReleases.flat()); return profile; } diff --git a/src/scrapers/score.js b/src/scrapers/score.js index 810f715f..b21b93b4 100644 --- a/src/scrapers/score.js +++ b/src/scrapers/score.js @@ -28,7 +28,7 @@ async function fetchPhotos(url) { return []; } -function scrapeAll(html) { +function scrapeAll(html, site) { return exa(html, '.container .video, .container-fluid .video').map(({ q, qa, qd, ql }) => { const release = {}; @@ -45,8 +45,8 @@ function scrapeAll(html) { release.date = qd('.i-date', 'MMM DD', /\w+ \d{1,2}$/) || qd('.dt-box', 'MMM.DD YYYY'); - release.actors = qa('.model, .i-model', true); - release.duration = ql('.i-amount'); + release.actors = site.parameters?.actors || qa('.model, .i-model', true); + release.duration = ql('.i-amount, .amount'); const posterEl = q('.item-img img'); @@ -64,20 +64,40 @@ function scrapeAll(html) { }).filter(Boolean); } -async function scrapeScene(html, url) { - const { q, qa, qtext, qi, qd, ql, qu, qis, qp, qt } = ex(html, '#videos-page, #content'); +async function scrapeScene(html, url, site) { + const { q, qa, qtext, qi, qd, ql, qu, qis, qp } = ex(html, '#videos-page, #content'); const release = {}; [release.entryId] = new URL(url).pathname.split('/').slice(-2); - release.title = q('h2.text-uppercase, h2.title, #breadcrumb-top + h1', true); + release.title = q('h2.text-uppercase, h2.title, #breadcrumb-top + h1', true) + || q('h1.m-title', true)?.split('ยป').slice(-1)[0].trim(); release.description = qtext('.p-desc, .desc'); release.actors = qa('.value a[href*=models], .value a[href*=performer], .value a[href*=teen-babes]', true); + + if (release.actors.length === 0) { + const actorEl = qa('.stat').find(stat => /Featuring/.test(stat.textContent)) + const actorString = qtext(actorEl); + + console.log(actorString); + /* + ?.split(/, and|,/g) + .map(actor => actor.trim()) + || []; + */ + } + + console.log(release.actors); + + if (release.actors.length === 0) release.actors = site.parameters?.actors; + release.tags = qa('a[href*=tag]', true); const dateEl = qa('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent)); - release.date = qd(dateEl, null, 'MMMM Do, YYYY'); + release.date = qd(dateEl, null, 'MMMM Do, YYYY') + || qd('.date', 'MMMM Do, YYYY', /\w+ \d{1,2}\w+, \d{4}/) + || qd('.info .holder', 'MM/DD/YYYY', /\d{2}\/\d{2}\/\d{4}/); const durationEl = qa('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent)); release.duration = ql(durationEl); @@ -94,25 +114,23 @@ async function scrapeScene(html, url) { ]); } - const trailer = qt(); + const trailers = qa('a[href*=Trailers]'); - if (trailer) { - release.trailer = [ - { - // don't rely on trailer always being 720p by default - src: trailer.replace(/\d+p\.mp4/, '720p.mp4'), - quality: 720, - }, - { - src: trailer.replace(/\d+p\.mp4/, '360p.mp4'), - quality: 360, - }, - ]; + if (trailers) { + release.trailer = trailers.map((trailer) => { + const src = `https:${trailer.href}`; + const format = trailer.textContent.trim().match(/^\w+/)[0].toLowerCase(); + const quality = parseInt(trailer.textContent.trim().match(/\d+([a-zA-Z]+)?$/)[0], 10); + + return format === 'mp4' ? { src, quality } : null; + }).filter(Boolean); } const stars = q('.rate-box').dataset.score; if (stars) release.rating = { stars }; + console.log(release); + return release; } diff --git a/src/sites.js b/src/sites.js index 3515f507..4a2ed0a4 100644 --- a/src/sites.js +++ b/src/sites.js @@ -60,21 +60,24 @@ function destructConfigNetworks(networks = []) { } async function findSiteByUrl(url) { - const { hostname } = new URL(url); - const domain = hostname.replace(/www.|tour./, ''); + const { origin, pathname } = new URL(url); + // const domain = hostname.replace(/www.|tour./, ''); + const dirUrl = `${origin}${pathname.split('/').slice(0, 2).join('/')}`; // allow for sites on URI directory - const sites = await knex('sites') + const site = await knex('sites') .leftJoin('networks', 'sites.network_id', 'networks.id') .select( 'sites.*', 'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters', ) - .where('sites.url', 'like', `%${domain}`) - .orWhere('sites.url', 'like', url) - .orWhere('sites.url', url); + .where('sites.url', url) + .orWhere('sites.url', origin) + .orWhere('sites.url', dirUrl) + // .orWhere('sites.url', 'like', `%${domain}`) + .first(); - if (sites.length > 0) { - const curatedSite = curateSite(sites[0], true); + if (site) { + const curatedSite = curateSite(site, true); return curatedSite; } diff --git a/src/utils/posters.js b/src/utils/posters.js index abf27365..4aab63ee 100644 --- a/src/utils/posters.js +++ b/src/utils/posters.js @@ -5,7 +5,6 @@ const path = require('path'); const fs = require('fs-extra'); const argv = require('../argv'); - const knex = require('../knex'); async function init() { @@ -26,6 +25,8 @@ async function init() { return file; })); + + knex.destroy(); } init(); diff --git a/src/utils/q.js b/src/utils/q.js index 4f22bb74..c7c073f7 100644 --- a/src/utils/q.js +++ b/src/utils/q.js @@ -2,6 +2,7 @@ const { JSDOM } = require('jsdom'); const moment = require('moment'); +const bhttp = require('bhttp'); function prefixProtocol(url, protocol = 'https') { if (protocol && /^\/\//.test(url)) { @@ -22,7 +23,7 @@ function q(context, selector, attrArg, trim = true) { return trim ? value?.trim() : value; } - return context.querySelector(selector); + return selector ? context.querySelector(selector) : context; } function qall(context, selector, attrArg, trim = true) { @@ -36,7 +37,7 @@ function qall(context, selector, attrArg, trim = true) { } function qtext(context, selector, trim = true) { - const el = q(context, selector, false, trim); + const el = q(context, selector, null, trim); if (!el) return null; const text = Array.from(el.childNodes) @@ -147,7 +148,7 @@ const funcs = { qus: qurls, }; -function ctx(element, window) { +function init(element, window) { if (!element) return null; const contextFuncs = Object.entries(funcs) // dynamically attach methods with context @@ -166,30 +167,58 @@ function ctx(element, window) { }; } -function ctxa(context, selector, window) { - return Array.from(context.querySelectorAll(selector)).map(element => ctx(element, window)); +function initAll(context, selector, window) { + return Array.from(context.querySelectorAll(selector)) + .map(element => init(element, window)); } -function ex(html, selector) { +function extract(html, selector) { const { window } = new JSDOM(html); if (selector) { - return ctx(window.document.querySelector(selector), window); + return init(window.document.querySelector(selector), window); } - return ctx(window.document, window); + return init(window.document, window); } -function exa(html, selector) { +function extractAll(html, selector) { const { window } = new JSDOM(html); - return ctxa(window.document, selector, window); + return initAll(window.document, selector, window); +} + +async function get(url, selector, headers, all = false) { + const res = await bhttp.get(url, { + headers, + }); + + if (res.statusCode === 200) { + return all + ? extractAll(res.body.toString(), selector) + : extract(res.body.toString(), selector); + } + + return null; +} + +async function getAll(url, selector, headers) { + return get(url, selector, headers, true); } module.exports = { - ex, - exa, - ctx, - ctxa, + extract, + extractAll, + init, + initAll, + get, + getAll, + context: init, + contextAll: initAll, + ex: extract, + exa: extractAll, + ctx: init, + ctxa: initAll, + geta: getAll, ...funcs, };