From 70e27a6cd93f2e2ba87dc50c9b8f68df78b30b7c Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Fri, 3 Jan 2020 00:59:02 +0100 Subject: [PATCH] Moved networks to GraphQL. --- assets/components/actors/actor.vue | 2 - assets/components/networks/network.vue | 22 +++--- assets/components/tile/release.vue | 3 +- assets/js/curate.js | 33 ++++++++- assets/js/fragments.js | 18 ++++- assets/js/networks/actions.js | 69 +++++++++++++++---- migrations/20190325001339_releases.js | 5 ++ public/css/style.css | 3 +- src/actors.js | 2 +- src/app.js | 12 +--- src/media.js | 35 +++++----- src/{scrape-release.js => scrape-releases.js} | 29 +++++--- src/scrape-sites.js | 8 +-- src/scrapers/freeones.js | 4 +- 14 files changed, 169 insertions(+), 76 deletions(-) rename src/{scrape-release.js => scrape-releases.js} (70%) diff --git a/assets/components/actors/actor.vue b/assets/components/actors/actor.vue index d215c258..5e68b2d1 100644 --- a/assets/components/actors/actor.vue +++ b/assets/components/actors/actor.vue @@ -254,8 +254,6 @@ function scrollPhotos(event) { async function mounted() { this.actor = await this.$store.dispatch('fetchActors', { actorSlug: this.$route.params.actorSlug }); - console.log(this.actor.releases[0]); - if (this.actor) { this.pageTitle = this.actor.name; } diff --git a/assets/components/networks/network.vue b/assets/components/networks/network.vue index e2399626..042855cb 100644 --- a/assets/components/networks/network.vue +++ b/assets/components/networks/network.vue @@ -3,7 +3,7 @@ v-if="network" class="content network" > - +
@@ -54,20 +54,18 @@ import FilterBar from '../header/filter-bar.vue'; import Releases from '../releases/releases.vue'; import SiteTile from '../tile/site.vue'; -async function fetchReleases() { - this.releases = await this.$store.dispatch('fetchNetworkReleases', this.$route.params.networkSlug); -} - -async function mounted() { - [[this.network]] = await Promise.all([ - this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug), - this.fetchReleases(), - ]); +async function fetchNetwork() { + this.network = await this.$store.dispatch('fetchNetworks', this.$route.params.networkSlug); this.sites = this.network.sites .filter(site => !site.independent) .sort(({ name: nameA }, { name: nameB }) => nameA.localeCompare(nameB)); + this.releases = this.network.sites.map(site => site.releases).flat(); +} + +async function mounted() { + await this.fetchNetwork(); this.pageTitle = this.network.name; } @@ -81,13 +79,13 @@ export default { return { network: null, sites: null, - releases: null, + releases: [], pageTitle: null, }; }, mounted, methods: { - fetchReleases, + fetchNetwork, }, }; diff --git a/assets/components/tile/release.vue b/assets/components/tile/release.vue index 33c3891e..88211841 100644 --- a/assets/components/tile/release.vue +++ b/assets/components/tile/release.vue @@ -249,9 +249,8 @@ export default { } .tags { - max-height: 2.5rem; + max-height: .5rem; padding: .25rem .5rem 1rem .5rem; - line-height: 1.5rem; word-wrap: break-word; overflow-y: hidden; } diff --git a/assets/js/curate.js b/assets/js/curate.js index 3398168e..77cfa770 100644 --- a/assets/js/curate.js +++ b/assets/js/curate.js @@ -12,8 +12,6 @@ function curateActor(actor) { } function curateRelease(release) { - console.log(release); - const curatedRelease = { ...release, actors: release.actors ? release.actors.map(({ actor }) => curateActor(actor)) : [], @@ -28,6 +26,35 @@ function curateRelease(release) { return curatedRelease; } +function curateSite(site, network) { + const curatedSite = { + id: site.id, + name: site.name, + slug: site.slug, + url: site.url, + }; + + if (site.releases) curatedSite.releases = site.releases.map(release => curateRelease(release)); + if (site.network || network) curatedSite.network = site.network || network; + + return curatedSite; +} + +function curateNetwork(network) { + const curatedNetwork = { + id: network.id, + name: network.name, + slug: network.slug, + url: network.url, + }; + + if (network.sites) { + curatedNetwork.sites = network.sites.map(site => curateSite(site, curatedNetwork)); + } + + return curatedNetwork; +} + function curateTag(tag) { const curatedTag = { ...tag, @@ -43,5 +70,7 @@ function curateTag(tag) { export { curateActor, curateRelease, + curateSite, + curateNetwork, curateTag, }; diff --git a/assets/js/fragments.js b/assets/js/fragments.js index 412f4e12..aac9e17b 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -13,6 +13,21 @@ const siteFragment = ` } `; +const sitesFragment = ` + sites { + id + name + slug + url + network { + id + name + slug + url + } + } +`; + const releaseActorsFragment = ` actors: releasesActorsSortables(orderBy: GENDER_ASC) { actor { @@ -36,7 +51,7 @@ const releaseActorsFragment = ` `; const releaseTagsFragment = ` - tags: releasesTags { + tags: releasesTagsSortables(orderBy: PRIORITY_DESC) { tag { name priority @@ -135,4 +150,5 @@ export { releasesFragment, releaseFragment, siteFragment, + sitesFragment, }; diff --git a/assets/js/networks/actions.js b/assets/js/networks/actions.js index 4ee3261a..675aaab6 100644 --- a/assets/js/networks/actions.js +++ b/assets/js/networks/actions.js @@ -1,27 +1,68 @@ -import { get } from '../api'; +import { graphql } from '../api'; +import { sitesFragment, releasesFragment } from '../fragments'; +import { curateNetwork } from '../curate'; function initNetworksActions(store, _router) { - async function fetchNetworks({ _commit }, networkId) { - const networks = await get(`/networks/${networkId || ''}`, { - - }); - - return networks; - } - - async function fetchNetworkReleases({ _commit }, networkId) { - const releases = await get(`/networks/${networkId}/releases`, { - filter: store.state.ui.filter, + async function fetchNetworkBySlug(networkSlug, limit = 100) { + const { network } = await graphql(` + query Network( + $networkSlug: String! + $limit:Int = 1000, + $after:Date = "1900-01-01", + $before:Date = "2100-01-01", + ) { + network: networkBySlug(slug: $networkSlug) { + id + name + slug + url + sites { + id + name + slug + url + ${releasesFragment} + network { + id + name + slug + url + } + } + } + } + `, { + networkSlug, + limit, after: store.getters.after, before: store.getters.before, }); - return releases; + return curateNetwork(network); + } + + async function fetchNetworks({ _commit }, networkSlug) { + if (networkSlug) { + return fetchNetworkBySlug(networkSlug); + } + + const { networks } = await graphql(` + query Networks { + networks { + id + name + slug + url + ${sitesFragment} + } + } + `); + + return networks.map(network => curateNetwork(network)); } return { fetchNetworks, - fetchNetworkReleases, }; } diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index f6f2fcef..5c440352 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -435,7 +435,12 @@ exports.up = knex => Promise.resolve() SELECT releases_actors.*, actors.gender, actors.name, actors.birthdate FROM releases_actors JOIN actors ON releases_actors.actor_id = actors.id; + CREATE VIEW releases_tags_sortable AS + SELECT releases_tags.*, tags.name, tags.priority FROM releases_tags + JOIN tags ON releases_tags.tag_id = tags.id; + COMMENT ON VIEW releases_actors_sortable IS E'@foreignKey (release_id) references releases (id)\n@foreignKey (actor_id) references actors (id)'; + COMMENT ON VIEW releases_tags_sortable IS E'@foreignKey (release_id) references releases (id)\n@foreignKey (tag_id) references tags (id)'; /* allow conversion resolver to be added for height and weight */ COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many'; diff --git a/public/css/style.css b/public/css/style.css index 770d34a6..5aeef339 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -215,9 +215,8 @@ line-height: 1.5rem; } .tags[data-v-3abcf101] { - max-height: 2.5rem; + max-height: .5rem; padding: .25rem .5rem 1rem .5rem; - line-height: 1.5rem; word-wrap: break-word; overflow-y: hidden; } diff --git a/src/actors.js b/src/actors.js index d2aa6dbf..9569c0df 100644 --- a/src/actors.js +++ b/src/actors.js @@ -370,7 +370,7 @@ async function scrapeActors(actorNames) { await createMediaDirectory('actors', `${newActorEntry.slug}/`); await storePhotos(profile.avatars, { - domain: 'actors', + domain: 'actor', role: 'photo', primaryRole: 'avatar', targetId: newActorEntry.id, diff --git a/src/app.js b/src/app.js index 61d2b4ec..5467d7b5 100644 --- a/src/app.js +++ b/src/app.js @@ -1,26 +1,20 @@ 'use strict'; -const Promise = require('bluebird'); - const argv = require('./argv'); const knex = require('./knex'); const initServer = require('./web/server'); const scrapeSites = require('./scrape-sites'); -const scrapeRelease = require('./scrape-release'); +const { scrapeReleases } = require('./scrape-releases'); const { scrapeActors, scrapeBasicActors } = require('./actors'); async function init() { if (argv.scene) { - await Promise.map(argv.scene, async url => scrapeRelease(url, null, false, 'scene'), { - concurrency: 5, - }); + await scrapeReleases(argv.scene, null, 'scene'); } if (argv.movie) { - await Promise.map(argv.movie, async url => scrapeRelease(url, null, false, 'movie'), { - concurrency: 5, - }); + await scrapeReleases(argv.movie, null, 'movie'); } if (argv.scrape || argv.networks || argv.sites) { diff --git a/src/media.js b/src/media.js index 1872e1c0..48acf1b2 100644 --- a/src/media.js +++ b/src/media.js @@ -48,6 +48,7 @@ async function createThumbnail(buffer) { } async function createMediaDirectory(domain, subpath) { + console.log(domain, subpath); const filepath = path.join(config.media.path, domain, subpath); await fs.mkdir(filepath, { recursive: true }); @@ -246,8 +247,6 @@ async function storeTrailer(trailers, { const [sourceDuplicates, sourceOriginals] = await findDuplicates([trailer], 'source', 'src', label); const metaFiles = await Promise.map(sourceOriginals, async (trailerX) => { - console.log('trailer x', trailerX, trailerX.src); - const { pathname } = new URL(trailerX.src); const mimetype = trailerX.type || mime.getType(pathname); @@ -256,6 +255,7 @@ async function storeTrailer(trailers, { const filepath = path.join(domain, subpath, `trailer${trailerX.quality ? `_${trailerX.quality}` : ''}.${mime.getExtension(mimetype)}`); return { + trailer: res.body, path: filepath, mime: mimetype, source: trailerX.src, @@ -264,25 +264,28 @@ async function storeTrailer(trailers, { }; }); - const [hashDuplicates, hashOriginals] = await findDuplicates(metaFiles, 'hash', null, label); - - console.log('hash dup', hashDuplicates, hashOriginals); + const [hashDuplicates, hashOriginals] = await findDuplicates(metaFiles, 'hash', 'hash', label); const newTrailers = await knex('media') - .insert(hashOriginals) + .insert(hashOriginals.map(trailerX => ({ + path: trailerX.path, + mime: trailerX.mime, + source: trailerX.source, + quality: trailerX.quality, + hash: trailerX.hash, + }))) .returning('*'); - console.log(newTrailers); + await Promise.all(hashOriginals.map(trailerX => fs.writeFile(path.join(config.media.path, trailerX.path), trailerX.trailer))); - await Promise.all([ - // fs.writeFile(path.join(config.media.path, filepath), res.body), - /* - knex('releases_trailers').insert({ - release_id: targetId, - media_id: mediaEntries[0].id, - }), - */ - ]); + const trailerEntries = Array.isArray(newTrailers) + ? [...sourceDuplicates, ...hashDuplicates, ...newTrailers] + : [...sourceDuplicates, ...hashDuplicates]; + + await upsert('releases_trailers', trailerEntries.map(trailerEntry => ({ + release_id: targetId, + media_id: trailerEntry.id, + })), ['release_id', 'media_id']); } module.exports = { diff --git a/src/scrape-release.js b/src/scrape-releases.js similarity index 70% rename from src/scrape-release.js rename to src/scrape-releases.js index d7d0a00d..d02ee4e9 100644 --- a/src/scrape-release.js +++ b/src/scrape-releases.js @@ -1,6 +1,7 @@ 'use strict'; const config = require('config'); +const Promise = require('bluebird'); const argv = require('./argv'); const scrapers = require('./scrapers/scrapers'); @@ -28,7 +29,7 @@ async function findSite(url, release) { return null; } -async function scrapeRelease(url, release, deep = true, type = 'scene') { +async function scrapeRelease(url, release, type = 'scene') { const site = await findSite(url, release); if (!site) { @@ -53,10 +54,17 @@ async function scrapeRelease(url, release, deep = true, type = 'scene') { ? await scraper.fetchScene(url, site, release) : await scraper.fetchMovie(url, site, release); - const curatedRelease = { ...scrapedRelease, type }; + return scrapedRelease; +} - if (!deep && argv.save) { - // don't store release when called by site scraper +async function scrapeReleases(urls, release, type = 'scene') { + const scrapedReleases = await Promise.map(urls, async url => scrapeRelease(url, release, type), { + concurrency: 5, + }); + + const curatedReleases = scrapedReleases.map(scrapedRelease => ({ ...scrapedRelease, type })); + + if (argv.save) { /* const movie = scrapedRelease.movie ? await scrapeRelease(scrapedRelease.movie, null, false, 'movie') @@ -68,14 +76,15 @@ async function scrapeRelease(url, release, deep = true, type = 'scene') { } */ - const { releases: [storedRelease] } = await storeReleases([curatedRelease]); + const { releases: storedReleases } = await storeReleases(curatedReleases); - if (storedRelease) { - console.log(`http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`); + if (storedReleases) { + console.log(storedReleases.map(storedRelease => `http://${config.web.host}:${config.web.port}/scene/${storedRelease.id}`).join('\n')); } } - - return scrapedRelease; } -module.exports = scrapeRelease; +module.exports = { + scrapeRelease, + scrapeReleases, +}; diff --git a/src/scrape-sites.js b/src/scrape-sites.js index 0859dbf7..f7cdf66d 100644 --- a/src/scrape-sites.js +++ b/src/scrape-sites.js @@ -7,7 +7,7 @@ const argv = require('./argv'); const knex = require('./knex'); const { fetchIncludedSites } = require('./sites'); const scrapers = require('./scrapers/scrapers'); -const scrapeRelease = require('./scrape-release'); +const { scrapeRelease } = require('./scrape-releases'); const { storeReleases } = require('./releases'); function getAfterDate() { @@ -70,7 +70,7 @@ async function deepFetchReleases(baseReleases) { return Promise.map(baseReleases, async (release) => { if (release.url) { try { - const fullRelease = await scrapeRelease(release.url, release, true, 'scene'); + const fullRelease = await scrapeRelease(release.url, release, 'scene'); return { ...release, @@ -111,7 +111,7 @@ async function scrapeSiteReleases(scraper, site) { return baseReleases; } -async function scrapeReleases() { +async function scrapeSites() { const networks = await fetchIncludedSites(); const scrapedNetworks = await Promise.map(networks, async network => Promise.map(network.sites, async (site) => { @@ -147,4 +147,4 @@ async function scrapeReleases() { } } -module.exports = scrapeReleases; +module.exports = scrapeSites; diff --git a/src/scrapers/freeones.js b/src/scrapers/freeones.js index 0f0a02bb..ff365e5f 100644 --- a/src/scrapers/freeones.js +++ b/src/scrapers/freeones.js @@ -50,7 +50,9 @@ function scrapeProfile(html, actorName) { if (bio.weight) profile.weight = Number(bio.weight.split(',')[0]); profile.social = Array.from(document.querySelectorAll('.profile-meta-item a.social-icons'), el => el.href); - profile.avatar = document.querySelector('.profile-image-large img').src; + + const avatar = document.querySelector('.profile-image-large img').src; + if (!avatar.match('placeholder')) profile.avatar = document.querySelector('.profile-image-large img').src; return profile; }