Added basic project structure and experimental entity fetching.

.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.js]
+charset = utf-8

.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+config/
+!config/default.*js
+dist/

.nvmrc

@@ -0,0 +1 @@
+v22.11.0

knexfile.js

@@ -0,0 +1,6 @@
+import config from 'config';
+
+export default {
+	client: 'pg',
+	connection: { ...config.get('database') }, // avoid "cannot redefine password" error
+};

package.json

@@ -1,22 +1,44 @@
 {
-  "name": "traxxx-core",
-  "version": "1.0.0",
-  "description": "Archiving core of the traxxx project.",
-  "main": "src/app.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://gitea.unknown.name/DebaucheryLibrarian/traxxx-core"
-  },
-  "keywords": [
-    "adult",
-    "nsfw",
-    "porn",
-    "xxx",
-    "archive"
-  ],
-  "author": "DebaucheryLibrarian",
-  "license": "ISC"
+	"name": "traxxx-core",
+	"version": "1.0.0",
+	"description": "Archiving core of the traxxx project.",
+	"main": "src/app.ts",
+	"type": "module",
+	"scripts": {
+		"postinstall": "node-config-ts",
+		"start": "tsx src/app.ts"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://gitea.unknown.name/DebaucheryLibrarian/traxxx-core"
+	},
+	"keywords": [
+		"adult",
+		"nsfw",
+		"porn",
+		"xxx",
+		"archive"
+	],
+	"author": "DebaucheryLibrarian",
+	"license": "ISC",
+	"devDependencies": {
+		"@types/config": "^3.3.5",
+		"@types/node": "^22.10.0",
+		"@types/yargs": "^17.0.33",
+		"eslint": "^9.15.0",
+		"typescript": "^5.7.2"
+	},
+	"dependencies": {
+		"config": "^3.3.12",
+		"knex": "^3.1.0",
+		"kysely": "^0.27.5",
+		"node-config-ts": "^3.3.1",
+		"pg": "^8.13.1",
+		"tsx": "^4.19.2",
+		"unprint": "^0.14.3",
+		"yargs": "^17.7.2"
+	},
+	"imports": {
+		"#src/*": "./dist/*.js"
+	}
 }

src/app.ts

@@ -0,0 +1,50 @@
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+
+import { update } from '#src/updates';
+
+console.log(process.argv);
+
+yargs(hideBin(process.argv))
+    .command('update', 'scan channels for scene and movie', (builder: yargs.Argv) => {
+		console.log('build update');
+		builder
+			.option('latest', {
+				describe: 'Scrape latest releases if available',
+				type: 'boolean',
+				default: true,
+			})
+			.option('upcoming', {
+				describe: 'Scrape upcoming releases if available',
+				type: 'boolean',
+				default: true,
+			})
+			.option('networks', {
+				describe: 'Network to scrape all channels from (overrides configuration)',
+				type: 'array',
+				alias: ['network'],
+			})
+			.option('exclude-networks', {
+				describe: 'Network not to scrape any channels from (overrides configuration)',
+				type: 'array',
+				alias: 'exclude-network',
+			})
+			.option('channels', {
+				describe: 'Channel to scrape (overrides configuration)',
+				type: 'array',
+				alias: ['channel'],
+			})
+			.option('exclude-channels', {
+				describe: 'Channel not to scrape (overrides configuration)',
+				type: 'array',
+				alias: 'exclude-channel',
+			})
+    }, async (argv: any) => update({
+		latest: argv.latest,
+		upcoming: argv.upcoming,
+		includeChannels: argv.channels,
+		includeNetworks: argv.networks,
+		excludeChannels: argv.excludeChannels,
+		excludeNetworks: argv.excludeNetworks,
+	}))
+	.parse();

src/entities.ts

@@ -0,0 +1,43 @@
+import knex from '#src/knex';
+
+type EntityEntry = {
+	id: number;
+	parent_id: number;
+	slug: string;
+	name: string;
+	type: string;
+};
+
+type BuildOptions = {
+	includeChannels?: string[];
+	includeNetworks?: string[];
+	excludeChannels?: string[];
+	excludeNetworks?: string[];
+};
+
+function buildEntityTree(entities: EntityEntry[], options: BuildOptions, branch: EntityEntry[], parent: EntityEntry | null): EntityEntry[] {
+	if (branch.length === 0) {
+		return [];
+	}
+
+	return branch.map((branchEntity) => ({
+		...branchEntity,
+		parent,
+		children: buildEntityTree(entities, options, entities.filter((entity) => entity.parent_id === branchEntity.id), branchEntity),
+	}));
+}
+
+function filterEntityTree(tree: EntityEntry[], options: BuildOptions) {
+	// travel bottom up, because include Team Skeet should also include its parent Team Skeet Media, but not its siblings
+}
+
+export async function buildEntities(options: BuildOptions = {}) {
+	const entities: EntityEntry[] = await knex<EntityEntry>('entities');
+	const entityTree = buildEntityTree(entities, options, entities.filter((entity) => entity.parent_id === null), null);
+	// TODO: possibly easier to filter the raw entities instead of the tree?!
+	// TODO: but I need to know what channels belong to what networks, include <channel> shouldn't filter its parent network
+	const filteredEntityTree = filterEntityTree(entityTree);
+
+	console.log(options);
+	console.log(entityTree);
+}

src/knex.ts

@@ -0,0 +1,10 @@
+import { config } from 'node-config-ts';
+import knex, { type Knex } from 'knex';
+
+export default knex({
+	client: 'pg',
+	connection: config.database,
+	// performance overhead, don't use asyncStackTraces in production
+	asyncStackTraces: process.env.NODE_ENV === 'development',
+	// debug: process.env.NODE_ENV === 'development',
+}) as Knex;

src/updates.ts

@@ -0,0 +1,19 @@
+import { buildEntities } from '#src/entities';
+
+export async function update(options: {
+	latest: boolean;
+	upcoming: boolean;
+	includeChannels?: string[];
+	includeNetworks?: string[];
+	excludeChannels?: string[];
+	excludeNetworks?: string[];
+}) {
+	console.log('UPDATE', options);
+
+	await buildEntities({
+		includeChannels: options.includeChannels,
+		includeNetworks: options.includeNetworks,
+		excludeChannels: options.excludeChannels,
+		excludeNetworks: options.excludeNetworks,
+	});
+}

tsconfig.json

@@ -0,0 +1,16 @@
+{
+	"compilerOptions": {
+		"strict": true,
+		"target": "es2022",
+		"module": "esnext",
+		"outDir": "dist",
+		"types": ["node"],
+		"esModuleInterop": true,
+		"forceConsistentCasingInFileNames": true,
+		"skipLibCheck": true,
+		"baseUrl": ".",
+		"paths": {
+			"#src/*": ["./src/*"]
+		}
+	}
+}