Added basic project structure and experimental entity fetching.
This commit is contained in:
parent
b7b3769dc8
commit
85c42f510d
|
@ -0,0 +1,14 @@
|
|||
# top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
# Unix-style newlines with a newline ending every file
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
|
||||
# Matches multiple files with brace expansion notation
|
||||
# Set default charset
|
||||
[*.js]
|
||||
charset = utf-8
|
|
@ -0,0 +1,4 @@
|
|||
node_modules/
|
||||
config/
|
||||
!config/default.*js
|
||||
dist/
|
|
@ -0,0 +1,6 @@
|
|||
import config from 'config';
|
||||
|
||||
export default {
|
||||
client: 'pg',
|
||||
connection: { ...config.get('database') }, // avoid "cannot redefine password" error
|
||||
};
|
File diff suppressed because it is too large
Load Diff
28
package.json
28
package.json
|
@ -2,9 +2,11 @@
|
|||
"name": "traxxx-core",
|
||||
"version": "1.0.0",
|
||||
"description": "Archiving core of the traxxx project.",
|
||||
"main": "src/app.js",
|
||||
"main": "src/app.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
"postinstall": "node-config-ts",
|
||||
"start": "tsx src/app.ts"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
@ -18,5 +20,25 @@
|
|||
"archive"
|
||||
],
|
||||
"author": "DebaucheryLibrarian",
|
||||
"license": "ISC"
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@types/config": "^3.3.5",
|
||||
"@types/node": "^22.10.0",
|
||||
"@types/yargs": "^17.0.33",
|
||||
"eslint": "^9.15.0",
|
||||
"typescript": "^5.7.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"config": "^3.3.12",
|
||||
"knex": "^3.1.0",
|
||||
"kysely": "^0.27.5",
|
||||
"node-config-ts": "^3.3.1",
|
||||
"pg": "^8.13.1",
|
||||
"tsx": "^4.19.2",
|
||||
"unprint": "^0.14.3",
|
||||
"yargs": "^17.7.2"
|
||||
},
|
||||
"imports": {
|
||||
"#src/*": "./dist/*.js"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,50 @@
|
|||
import yargs from 'yargs';
|
||||
import { hideBin } from 'yargs/helpers';
|
||||
|
||||
import { update } from '#src/updates';
|
||||
|
||||
console.log(process.argv);
|
||||
|
||||
yargs(hideBin(process.argv))
|
||||
.command('update', 'scan channels for scene and movie', (builder: yargs.Argv) => {
|
||||
console.log('build update');
|
||||
builder
|
||||
.option('latest', {
|
||||
describe: 'Scrape latest releases if available',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
})
|
||||
.option('upcoming', {
|
||||
describe: 'Scrape upcoming releases if available',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
})
|
||||
.option('networks', {
|
||||
describe: 'Network to scrape all channels from (overrides configuration)',
|
||||
type: 'array',
|
||||
alias: ['network'],
|
||||
})
|
||||
.option('exclude-networks', {
|
||||
describe: 'Network not to scrape any channels from (overrides configuration)',
|
||||
type: 'array',
|
||||
alias: 'exclude-network',
|
||||
})
|
||||
.option('channels', {
|
||||
describe: 'Channel to scrape (overrides configuration)',
|
||||
type: 'array',
|
||||
alias: ['channel'],
|
||||
})
|
||||
.option('exclude-channels', {
|
||||
describe: 'Channel not to scrape (overrides configuration)',
|
||||
type: 'array',
|
||||
alias: 'exclude-channel',
|
||||
})
|
||||
}, async (argv: any) => update({
|
||||
latest: argv.latest,
|
||||
upcoming: argv.upcoming,
|
||||
includeChannels: argv.channels,
|
||||
includeNetworks: argv.networks,
|
||||
excludeChannels: argv.excludeChannels,
|
||||
excludeNetworks: argv.excludeNetworks,
|
||||
}))
|
||||
.parse();
|
|
@ -0,0 +1,43 @@
|
|||
import knex from '#src/knex';
|
||||
|
||||
type EntityEntry = {
|
||||
id: number;
|
||||
parent_id: number;
|
||||
slug: string;
|
||||
name: string;
|
||||
type: string;
|
||||
};
|
||||
|
||||
type BuildOptions = {
|
||||
includeChannels?: string[];
|
||||
includeNetworks?: string[];
|
||||
excludeChannels?: string[];
|
||||
excludeNetworks?: string[];
|
||||
};
|
||||
|
||||
function buildEntityTree(entities: EntityEntry[], options: BuildOptions, branch: EntityEntry[], parent: EntityEntry | null): EntityEntry[] {
|
||||
if (branch.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return branch.map((branchEntity) => ({
|
||||
...branchEntity,
|
||||
parent,
|
||||
children: buildEntityTree(entities, options, entities.filter((entity) => entity.parent_id === branchEntity.id), branchEntity),
|
||||
}));
|
||||
}
|
||||
|
||||
function filterEntityTree(tree: EntityEntry[], options: BuildOptions) {
|
||||
// travel bottom up, because include Team Skeet should also include its parent Team Skeet Media, but not its siblings
|
||||
}
|
||||
|
||||
export async function buildEntities(options: BuildOptions = {}) {
|
||||
const entities: EntityEntry[] = await knex<EntityEntry>('entities');
|
||||
const entityTree = buildEntityTree(entities, options, entities.filter((entity) => entity.parent_id === null), null);
|
||||
// TODO: possibly easier to filter the raw entities instead of the tree?!
|
||||
// TODO: but I need to know what channels belong to what networks, include <channel> shouldn't filter its parent network
|
||||
const filteredEntityTree = filterEntityTree(entityTree);
|
||||
|
||||
console.log(options);
|
||||
console.log(entityTree);
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
import { config } from 'node-config-ts';
|
||||
import knex, { type Knex } from 'knex';
|
||||
|
||||
export default knex({
|
||||
client: 'pg',
|
||||
connection: config.database,
|
||||
// performance overhead, don't use asyncStackTraces in production
|
||||
asyncStackTraces: process.env.NODE_ENV === 'development',
|
||||
// debug: process.env.NODE_ENV === 'development',
|
||||
}) as Knex;
|
|
@ -0,0 +1,19 @@
|
|||
import { buildEntities } from '#src/entities';
|
||||
|
||||
export async function update(options: {
|
||||
latest: boolean;
|
||||
upcoming: boolean;
|
||||
includeChannels?: string[];
|
||||
includeNetworks?: string[];
|
||||
excludeChannels?: string[];
|
||||
excludeNetworks?: string[];
|
||||
}) {
|
||||
console.log('UPDATE', options);
|
||||
|
||||
await buildEntities({
|
||||
includeChannels: options.includeChannels,
|
||||
includeNetworks: options.includeNetworks,
|
||||
excludeChannels: options.excludeChannels,
|
||||
excludeNetworks: options.excludeNetworks,
|
||||
});
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"strict": true,
|
||||
"target": "es2022",
|
||||
"module": "esnext",
|
||||
"outDir": "dist",
|
||||
"types": ["node"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"#src/*": ["./src/*"]
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue