boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Separated actor expand buttons. Refactored Brazzers scraper. Fixed actor releases not included in shallow scrape. Added number query and data-src default to qu img. Updated README. Removed post-install migrate and seed.

Browse Source
This commit is contained in:
ThePendulum
2020-07-09 02:00:54 +02:00
parent 17d46e804e
commit 44a8ced30c
10 changed files with 289 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/entities.js
View File

@@ -85,7 +85,54 @@ async function fetchChannelsFromArgv() {
}
async function fetchChannelsFromConfig() {
const rawSites = await knex('entities')
const rawNetworks = await knex.raw(`
WITH RECURSIVE children AS (
SELECT
id, parent_id, name, slug, type, url, description, parameters
FROM
entities
WHERE
CASE WHEN array_length(?, 1) IS NOT NULL
THEN slug = ANY(?)
ELSE true
END
AND NOT
slug = ANY(?)
AND
entities.type = 'network'
UNION ALL
SELECT
entities.id, entities.parent_id, entities.name, entities.slug, entities.type, entities.url, entities.description, entities.parameters
FROM
entities
INNER JOIN
children ON children.id = entities.parent_id
)
SELECT
entities.*, row_to_json(parents) as parent, json_agg(children) as children
FROM
children
LEFT JOIN
entities ON entities.id = children.parent_id
LEFT JOIN
entities AS parents ON parents.id = entities.parent_id
WHERE
children.type = 'channel'
GROUP BY
children.parent_id, entities.id, entities.name, parents.id
`, [
config.include.networks,
config.include.networks,
config.exclude.networks,
]);
console.log(rawNetworks.rows);
/*
const curatedSites = await curateEntities(rawChannels, true);
logger.info(`Found ${curatedSites.length} entities in database`);
const rawChannels = await knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent'))
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
.where((builder) => {
@@ -97,10 +144,10 @@ async function fetchChannelsFromConfig() {
builder.whereIn('entities.slug', config.exclude || []);
});
const curatedSites = await curateEntities(rawSites, true);
logger.info(`Found ${curatedSites.length} entities in database`);
console.log(rawChannels);
*/
return curatedSites;
// return curatedSites;
}
async function fetchIncludedEntities() {