diff --git a/config/default.js b/config/default.js index b35b32e2..0eda6b49 100644 --- a/config/default.js +++ b/config/default.js @@ -11,12 +11,18 @@ module.exports = { sfwHost: '0.0.0.0', sfwPort: 5001, }, - // include: [], + include: { + networks: [ + 'xempire', + 'julesjordan', + ], + channels: [], + }, exclude: { networks: [ - 'gamma', + 'hardx', + 'pornpros', 'mindgeek', - 'julesjordan', ], channels: [ // 21sextreme, no longer updated diff --git a/src/app.js b/src/app.js index b984e78e..eb9aca00 100644 --- a/src/app.js +++ b/src/app.js @@ -28,7 +28,7 @@ async function init() { const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames); const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.releases).flat().filter(Boolean); - const updateBaseScenes = (argv.all || argv.channels || argv.networks || argv.movies) && await fetchUpdates(); + const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates(); const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile); const sceneUrls = (argv.scene || []).concat(scenesFromFile || []); diff --git a/src/argv.js b/src/argv.js index ab533f1f..2bb12b7d 100644 --- a/src/argv.js +++ b/src/argv.js @@ -61,13 +61,12 @@ const { argv } = yargs .option('actors-scenes', { describe: 'Fetch all scenes for an actor', type: 'boolean', - alias: 'actor-scenes', default: false, }) .option('actors-sources', { describe: 'Use these scrapers for actor data', type: 'array', - alias: 'actor-source', + alias: 'source', }) .option('movie-scenes', { describe: 'Fetch all scenes for a movie', @@ -83,7 +82,6 @@ const { argv } = yargs .option('scene-actors', { describe: 'Scrape profiles for new actors after fetching scenes', type: 'boolean', - alias: 'with-profiles', default: false, }) .option('scene', { diff --git a/src/entities.js b/src/entities.js index 3f762529..b014b9f3 100644 --- a/src/entities.js +++ b/src/entities.js @@ -1,5 +1,6 @@ 'use strict'; +const util = require('util'); const config = require('config'); const logger = require('./logger')(__filename); @@ -85,6 +86,9 @@ async function fetchChannelsFromArgv() { } async function fetchChannelsFromConfig() { + console.log(config.include); + + /* const rawNetworks = await knex.raw(` WITH RECURSIVE children AS ( SELECT @@ -125,8 +129,71 @@ async function fetchChannelsFromConfig() { config.include.networks, config.exclude.networks, ]); + */ - console.log(rawNetworks.rows); + const rawNetworks = await knex.raw(` + /* select channels associated to configured networks */ + WITH RECURSIVE channels AS ( + /* select configured networks */ + SELECT + id, parent_id, name, type, slug + FROM + entities + WHERE + (slug = ANY(:includeNetworks) + AND NOT entities.slug = ANY(:excludedNetworks)) + AND entities.type = 'network' + + UNION ALL + + /* select recursive children of configured networks */ + SELECT + entities.id, entities.parent_id, entities.name, entities.type, entities.slug + FROM + entities + INNER JOIN + channels ON channels.id = entities.parent_id + WHERE + NOT ( + (entities.slug = ANY(:excludedNetworks) AND entities.type = 'network') + OR (entities.slug = ANY(:excludedChannels) AND entities.type = 'channel') + ) + ) + /* select recursive channels as children of networks */ + SELECT + entities.*, json_agg(channels) as children + FROM + channels + LEFT JOIN + entities ON entities.id = channels.parent_id + WHERE + channels.type = 'channel' + GROUP BY + entities.id + + UNION ALL + + /* select configured channels as children of networks */ + SELECT + entities.*, json_agg(children) as children + FROM + entities AS children + LEFT JOIN + entities ON entities.id = children.parent_id + WHERE + children.slug = ANY(:includedChannels) + AND + children.type = 'channel' + GROUP BY + entities.id + `, { + includedNetworks: config.include.networks, + includedChannels: config.include.channels, + excludedNetworks: config.exclude.networks, + excludedChannels: config.exclude.channels, + }); + + console.log(util.inspect(rawNetworks.rows, null, null)); /* const curatedSites = await curateEntities(rawChannels, true); diff --git a/src/releases.js b/src/releases.js index 36f1803b..ed40e52b 100644 --- a/src/releases.js +++ b/src/releases.js @@ -61,7 +61,7 @@ function curateRelease(release, withMedia = false) { }; } -function withRelations(queryBuilder, withMedia = false, type = 'scene') { +function withRelations(queryBuilder, withMedia = false) { queryBuilder .select(knex.raw(` releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at, @@ -70,7 +70,6 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') { COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags `)) - .where('releases.type', type) .leftJoin('entities', 'entities.id', 'releases.entity_id') .leftJoin('entities as parents', 'parents.id', 'entities.parent_id') .leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id') @@ -96,27 +95,27 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') { } } -async function fetchRelease(releaseId, type = 'scene') { +async function fetchRelease(releaseId) { const release = await knex('releases') .where('releases.id', releaseId) - .modify(withRelations, true, type) + .modify(withRelations, true) .first(); return curateRelease(release, true); } -async function fetchReleases(limit = 100, type = 'scene') { +async function fetchReleases(limit = 100) { const releases = await knex('releases') - .modify(withRelations, false, type) + .modify(withRelations, false) .limit(Math.min(limit, 1000)); return releases.map(release => curateRelease(release)); } -async function searchReleases(query, limit = 100, type = 'scene') { +async function searchReleases(query, limit = 100) { const releases = await knex .from(knex.raw('search_releases(?) as releases', [query])) - .modify(withRelations, false, type) + .modify(withRelations, false) .limit(Math.min(limit, 1000)); return releases.map(release => curateRelease(release));