DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added children to deep release entity, removed database dependency from Perfect Gonzo scraper.

This commit is contained in:
DebaucheryLibrarian 2021-02-02 01:51:22 +01:00
parent ab83dd2e55
commit 6364912aa8
2 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/entities.js
View File

@ -199,7 +199,11 @@ async function fetchReleaseEntities(baseReleases) {
FROM entity_tree FROM entity_tree
JOIN entities ON entity_tree.parent_id = entities.id JOIN entities ON entity_tree.parent_id = entities.id
) )
SELECT entity FROM entity_tree WHERE parent_id is null SELECT jsonb_set(entity, '{children}', to_jsonb(COALESCE(json_agg(children) FILTER (WHERE children.id IS NOT NULL), '[]'))) entity
FROM entity_tree
LEFT JOIN entities AS children ON children.parent_id = (entity->>'id')::int
WHERE entity_tree.parent_id IS NULL
GROUP BY entity_tree.entity
ORDER BY entity->'type' ASC; ORDER BY entity->'type' ASC;
`, { entitySlugs }); `, { entitySlugs });

12
src/scrapers/perfectgonzo.js
View File

@ -1,17 +1,9 @@
'use strict'; 'use strict';
const blake2 = require('blake2'); const blake2 = require('blake2');
const knex = require('../knex');
const qu = require('../utils/qu'); const qu = require('../utils/qu');
async function getSiteSlugs() {
return knex('entities')
.pluck('entities.slug')
.join('entities AS parents', 'parents.id', 'entities.parent_id')
.where('parents.slug', 'perfectgonzo');
}
function getHash(identifier) { function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 }); const hash = blake2.createHash('blake2b', { digestLength: 8 });
@ -39,9 +31,7 @@ function extractMaleModelsFromTags(tagContainer) {
} }
async function extractChannelFromPhoto(photo, channel) { async function extractChannelFromPhoto(photo, channel) {
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug) const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug);
|| await getSiteSlugs();
const channelMatch = photo.match(new RegExp(siteSlugs.join('|'))); const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
if (channelMatch) { if (channelMatch) {