boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Removed superfluous MindGeek scrapers.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-02-03 00:46:59 +01:00
parent 8337ce8dbd
commit 6d93083581
20 changed files with 62 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/entities.js
View File

@@ -167,15 +167,7 @@ async function fetchIncludedEntities() {
return curatedNetworks;
}
async function fetchReleaseEntities(baseReleases) {
const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity);
const entitySlugs = Array.from(new Set(
baseReleasesWithoutEntity
.map(baseRelease => urlToSiteSlug(baseRelease.url))
.filter(Boolean),
));
async function fetchEntitiesBySlug(entitySlugs, sort = 'asc') {
const entities = await knex.raw(`
WITH RECURSIVE entity_tree as (
SELECT to_jsonb(entities) as entity,
@@ -197,8 +189,8 @@ async function fetchReleaseEntities(baseReleases) {
LEFT JOIN entities AS children ON children.parent_id = (entity->>'id')::int
WHERE entity_tree.parent_id IS NULL
GROUP BY entity_tree.entity
ORDER BY entity->'type' ASC;
`, { entitySlugs });
ORDER BY entity->'type' :sort;
`, { entitySlugs, sort: knex.raw(sort) });
// channel entity will overwrite network entity
const entitiesBySlug = entities.rows.reduce((accEntities, { entity }) => ({
@@ -209,6 +201,18 @@ async function fetchReleaseEntities(baseReleases) {
return entitiesBySlug;
}
async function fetchReleaseEntities(baseReleases) {
const baseReleasesWithoutEntity = baseReleases.filter(release => release.url && !release.site && !release.entity);
const entitySlugs = Array.from(new Set(
baseReleasesWithoutEntity
.map(baseRelease => urlToSiteSlug(baseRelease.url))
.filter(Boolean),
));
return fetchEntitiesBySlug(entitySlugs);
}
async function fetchEntity(entityId, type) {
const entity = await knex('entities')
.select(knex.raw(`
@@ -361,6 +365,7 @@ module.exports = {
curateEntities,
fetchIncludedEntities,
fetchReleaseEntities,
fetchEntitiesBySlug,
fetchEntity,
fetchEntities,
searchEntities,