boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added children to deep release entity, removed database dependency from Perfect Gonzo scraper.

This commit is contained in:
DebaucheryLibrarian 2021-02-02 01:51:22 +01:00
parent ab83dd2e55
commit 6364912aa8
2 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/entities.js
View File

@ -199,7 +199,11 @@ async function fetchReleaseEntities(baseReleases) {
FROM entity_tree
JOIN entities ON entity_tree.parent_id = entities.id
)
SELECT entity FROM entity_tree WHERE parent_id is null
SELECT jsonb_set(entity, '{children}', to_jsonb(COALESCE(json_agg(children) FILTER (WHERE children.id IS NOT NULL), '[]'))) entity
FROM entity_tree
LEFT JOIN entities AS children ON children.parent_id = (entity->>'id')::int
WHERE entity_tree.parent_id IS NULL
GROUP BY entity_tree.entity
ORDER BY entity->'type' ASC;
`, { entitySlugs });

12
src/scrapers/perfectgonzo.js
View File

@ -1,17 +1,9 @@
'use strict';
const blake2 = require('blake2');
const knex = require('../knex');
const qu = require('../utils/qu');
async function getSiteSlugs() {
return knex('entities')
.pluck('entities.slug')
.join('entities AS parents', 'parents.id', 'entities.parent_id')
.where('parents.slug', 'perfectgonzo');
}
function getHash(identifier) {
const hash = blake2.createHash('blake2b', { digestLength: 8 });
@ -39,9 +31,7 @@ function extractMaleModelsFromTags(tagContainer) {
}
async function extractChannelFromPhoto(photo, channel) {
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug)
|| await getSiteSlugs();
const siteSlugs = (channel.type === 'network' ? channel.children : channel.parent?.children)?.map(child => child.slug);
const channelMatch = photo.match(new RegExp(siteSlugs.join('|')));
if (channelMatch) {