DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Removed superfluous MindGeek scrapers.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-02-03 00:46:59 +01:00
parent 8337ce8dbd
commit 6d93083581
20 changed files with 62 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/actors.js
View File

@@ -23,6 +23,7 @@ const logger = require('./logger')(__filename);
const { toBaseReleases } = require('./deep');
const { associateAvatars, flushOrphanedMedia } = require('./media');
const { fetchEntitiesBySlug } = require('./entities');
const { deleteScenes } = require('./releases');
const slugify = require('./utils/slugify');
@@ -740,23 +741,14 @@ async function scrapeActors(argNames) {
const sources = argv.profileSources || config.profiles || Object.keys(scrapers.actors);
const entitySlugs = sources.flat();
const [entities, existingActorEntries] = await Promise.all([
knex('entities')
.select(knex.raw('entities.*, row_to_json(parents) as parent, json_agg(children) as children'))
.whereIn('entities.slug', entitySlugs)
.whereIn('entities.type', ['network', 'channel'])
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
.leftJoin('entities as children', 'children.parent_id', 'entities.id')
.orderBy('entities.type')
.groupBy('entities.id', 'parents.id'),
const [entitiesBySlug, existingActorEntries] = await Promise.all([
fetchEntitiesBySlug(entitySlugs, 'desc'),
knex('actors')
.select(['id', 'name', 'slug', 'entry_id'])
.whereIn('slug', baseActors.map(baseActor => baseActor.slug))
.whereNull('alias_for'),
]);
const entitiesBySlug = entities.reduce((acc, entity) => ({ ...acc, [entity.slug]: acc[entity.slug] || entity }), {});
const existingActorEntriesBySlugAndEntryId = existingActorEntries.reduce((acc, actorEntry) => ({
...acc,
[actorEntry.slug]: {