Removed type property from scenes API.

config/default.js

@@ -11,12 +11,18 @@ module.exports = {
 		sfwHost: '0.0.0.0',
 		sfwPort: 5001,
 	},
-	// include: [],
+	include: {
+		networks: [
+			'xempire',
+			'julesjordan',
+		],
+		channels: [],
+	},
 	exclude: {
 		networks: [
-			'gamma',
+			'hardx',
+			'pornpros',
 			'mindgeek',
-			'julesjordan',
 		],
 		channels: [
 			// 21sextreme, no longer updated

src/app.js

@@ -28,7 +28,7 @@ async function init() {
 	const actors = (argv.actors || argv.actorsUpdate || argv.actorsFile) && await scrapeActors(actorNames);
 	const actorBaseScenes = argv.actors && argv.actorScenes && actors.map(actor => actor.releases).flat().filter(Boolean);
 
-	const updateBaseScenes = (argv.all || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
+	const updateBaseScenes = (argv.latest || argv.upcoming || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
 
 	const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
 	const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);

src/argv.js

@@ -61,13 +61,12 @@ const { argv } = yargs
 	.option('actors-scenes', {
 		describe: 'Fetch all scenes for an actor',
 		type: 'boolean',
-		alias: 'actor-scenes',
 		default: false,
 	})
 	.option('actors-sources', {
 		describe: 'Use these scrapers for actor data',
 		type: 'array',
-		alias: 'actor-source',
+		alias: 'source',
 	})
 	.option('movie-scenes', {
 		describe: 'Fetch all scenes for a movie',
@@ -83,7 +82,6 @@ const { argv } = yargs
 	.option('scene-actors', {
 		describe: 'Scrape profiles for new actors after fetching scenes',
 		type: 'boolean',
-		alias: 'with-profiles',
 		default: false,
 	})
 	.option('scene', {

src/entities.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const util = require('util');
 const config = require('config');
 
 const logger = require('./logger')(__filename);
@@ -85,6 +86,9 @@ async function fetchChannelsFromArgv() {
 }
 
 async function fetchChannelsFromConfig() {
+	console.log(config.include);
+
+	/*
 	const rawNetworks = await knex.raw(`
 		WITH RECURSIVE children AS (
 			SELECT
@@ -125,8 +129,71 @@ async function fetchChannelsFromConfig() {
 		config.include.networks,
 		config.exclude.networks,
 	]);
+	*/
 
-	console.log(rawNetworks.rows);
+	const rawNetworks = await knex.raw(`
+		/* select channels associated to configured networks */
+		WITH RECURSIVE channels AS (
+			/* select configured networks */
+			SELECT
+				id, parent_id, name, type, slug
+			FROM
+				entities
+			WHERE
+				(slug = ANY(:includeNetworks)
+				AND NOT entities.slug = ANY(:excludedNetworks))
+				AND entities.type = 'network'
+
+			UNION ALL
+
+			/* select recursive children of configured networks */
+			SELECT
+				entities.id, entities.parent_id, entities.name, entities.type, entities.slug
+			FROM
+				entities
+			INNER JOIN
+				channels ON channels.id = entities.parent_id
+			WHERE
+				NOT (
+				(entities.slug = ANY(:excludedNetworks) AND entities.type = 'network')
+				OR (entities.slug = ANY(:excludedChannels) AND entities.type = 'channel')
+				)
+		)
+		/* select recursive channels as children of networks */
+		SELECT
+			entities.*, json_agg(channels) as children
+		FROM
+			channels
+		LEFT JOIN
+			entities ON entities.id = channels.parent_id
+		WHERE
+			channels.type = 'channel'
+		GROUP BY
+			entities.id
+
+		UNION ALL
+
+		/* select configured channels as children of networks */
+		SELECT
+			entities.*, json_agg(children) as children
+		FROM
+			entities AS children
+		LEFT JOIN
+			entities ON entities.id = children.parent_id
+		WHERE
+			children.slug = ANY(:includedChannels)
+		AND
+			children.type = 'channel'
+		GROUP BY
+			entities.id
+	`, {
+		includedNetworks: config.include.networks,
+		includedChannels: config.include.channels,
+		excludedNetworks: config.exclude.networks,
+		excludedChannels: config.exclude.channels,
+	});
+
+	console.log(util.inspect(rawNetworks.rows, null, null));
 
 	/*
 	const curatedSites = await curateEntities(rawChannels, true);

src/releases.js

@@ -61,7 +61,7 @@ function curateRelease(release, withMedia = false) {
 	};
 }
 
-function withRelations(queryBuilder, withMedia = false, type = 'scene') {
+function withRelations(queryBuilder, withMedia = false) {
 	queryBuilder
 		.select(knex.raw(`
 			releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
@@ -70,7 +70,6 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
 			COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
 			COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
 		`))
-		.where('releases.type', type)
 		.leftJoin('entities', 'entities.id', 'releases.entity_id')
 		.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
 		.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
@@ -96,27 +95,27 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
 	}
 }
 
-async function fetchRelease(releaseId, type = 'scene') {
+async function fetchRelease(releaseId) {
 	const release = await knex('releases')
 		.where('releases.id', releaseId)
-		.modify(withRelations, true, type)
+		.modify(withRelations, true)
 		.first();
 
 	return curateRelease(release, true);
 }
 
-async function fetchReleases(limit = 100, type = 'scene') {
+async function fetchReleases(limit = 100) {
 	const releases = await knex('releases')
-		.modify(withRelations, false, type)
+		.modify(withRelations, false)
 		.limit(Math.min(limit, 1000));
 
 	return releases.map(release => curateRelease(release));
 }
 
-async function searchReleases(query, limit = 100, type = 'scene') {
+async function searchReleases(query, limit = 100) {
 	const releases = await knex
 		.from(knex.raw('search_releases(?) as releases', [query]))
-		.modify(withRelations, false, type)
+		.modify(withRelations, false)
 		.limit(Math.min(limit, 1000));
 
 	return releases.map(release => curateRelease(release));