From 6364912aa84568651fd1aa38b86583354f30683d Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Tue, 2 Feb 2021 01:51:22 +0100 Subject: [PATCH] Added children to deep release entity, removed database dependency from Perfect Gonzo scraper. --- src/entities.js | 6 +++++- src/scrapers/perfectgonzo.js | 12 +----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/entities.js b/src/entities.js index 405d12f7..7e4b31e9 100644 --- a/src/entities.js +++ b/src/entities.js @@ -199,7 +199,11 @@ async function fetchReleaseEntities(baseReleases) { FROM entity_tree JOIN entities ON entity_tree.parent_id = entities.id ) - SELECT entity FROM entity_tree WHERE parent_id is null + SELECT jsonb_set(entity, '{children}', to_jsonb(COALESCE(json_agg(children) FILTER (WHERE children.id IS NOT NULL), '[]'))) entity + FROM entity_tree + LEFT JOIN entities AS children ON children.parent_id = (entity->>'id')::int + WHERE entity_tree.parent_id IS NULL + GROUP BY entity_tree.entity ORDER BY entity->'type' ASC; `, { entitySlugs }); diff --git a/src/scrapers/perfectgonzo.js b/src/scrapers/perfectgonzo.js index 4564bfc6..d5c460b0 100644 --- a/src/scrapers/perfectgonzo.js +++ b/src/scrapers/perfectgonzo.js @@ -1,17 +1,9 @@ 'use strict'; const blake2 = require('blake2'); -const knex = require('../knex'); const qu = require('../utils/qu'); -async function getSiteSlugs() { - return knex('entities') - .pluck('entities.slug') - .join('entities AS parents', 'parents.id', 'entities.parent_id') - .where('parents.slug', 'perfectgonzo'); -} - function getHash(identifier) { const hash = blake2.createHash('blake2b', { digestLength: 8 }); @@ -39,9 +31,7 @@ function extractMaleModelsFromTags(tagContainer) { } async function extractChannelFromPhoto(photo, channel) { - const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug) - || await getSiteSlugs(); - + const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug); const channelMatch = photo.match(new RegExp(siteSlugs.join('|'))); if (channelMatch) {