Users now go through profile, posts, content info and downloads one by one (preparing the next user while the current one downloads), saving each index as soon as that user is done. The queue is re-read between users, users are removed once archived, and a lock file lets a second --queue invocation hand its users to the running process. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
154 lines
5.8 KiB
JavaScript
154 lines
5.8 KiB
JavaScript
'use strict';
|
|
|
|
const config = require('config');
|
|
const yargs = require('yargs');
|
|
|
|
function getArgs() {
|
|
const args = yargs
|
|
.command('npm start -- --user <username>')
|
|
.option('log-level', {
|
|
alias: 'level',
|
|
describe: 'CLI log verbosity',
|
|
type: 'string',
|
|
default: config.logger.level,
|
|
})
|
|
.option('interval', {
|
|
describe: 'Minimum wait time between HTTP requests',
|
|
type: 'number',
|
|
default: config.limiter.interval,
|
|
})
|
|
.option('concurrency', {
|
|
describe: 'Maximum HTTP requests pending at the same time',
|
|
type: 'number',
|
|
default: config.limiter.concurrency,
|
|
})
|
|
.option('users', {
|
|
alias: 'user',
|
|
describe: 'Reddit usernames to fetch posts from',
|
|
type: 'array',
|
|
})
|
|
.option('file-users', {
|
|
describe: 'Load reddit usernames from file',
|
|
type: 'string',
|
|
})
|
|
.option('queue', {
|
|
describe: 'Work through a queue file of usernames (default: queue.file in config, \'queue\'), removing each user once archived and re-reading the file for new users as it goes. Users given with --users/--file-users are added to the queue. If another process is already working through the queue, only adds them and exits',
|
|
type: 'string',
|
|
})
|
|
.option('wait', {
|
|
describe: 'With --queue: keep running when the queue is empty, checking it for new users every queue.interval ms',
|
|
type: 'boolean',
|
|
})
|
|
.option('cleanup', {
|
|
describe: 'Only check the supplied users for suspended/deleted accounts: record them, and remove them from --file-users once dead for longer than configured. Does not archive anything',
|
|
type: 'boolean',
|
|
})
|
|
.option('posts', {
|
|
alias: 'post',
|
|
describe: 'Reddit post IDs to fetch',
|
|
type: 'array',
|
|
})
|
|
.option('file-posts', {
|
|
describe: 'Load reddit post IDs from file',
|
|
type: 'string',
|
|
})
|
|
.option('direct', {
|
|
describe: 'Get content directly from imgur and other hosts',
|
|
type: 'array',
|
|
alias: 'fetch',
|
|
})
|
|
.option('file-direct', {
|
|
describe: 'Load direct content URLs from file',
|
|
type: 'string',
|
|
alias: 'file-fetch',
|
|
})
|
|
.option('label', {
|
|
describe: 'Arbitrary variable made available in path patterns. Useful to organize files from a URL lists in directory.',
|
|
type: 'string',
|
|
})
|
|
.option('base', {
|
|
describe: 'Alternative base path, overriding both the default posts and direct base paths.',
|
|
type: 'string',
|
|
})
|
|
.option('limit', {
|
|
describe: 'Maximum amount of posts to fetch per supplied user (!), after filtering out ignored, cross- and reposts',
|
|
type: 'number',
|
|
default: config.fetch.limit,
|
|
})
|
|
.option('sort', {
|
|
describe: 'Property to sort posts by',
|
|
choices: ['new', 'top', 'hot', 'controversial'],
|
|
default: config.fetch.sort,
|
|
})
|
|
.option('ignore', {
|
|
describe: 'Ignore posts with any of these properties',
|
|
type: 'array',
|
|
choices: ['pinned', 'stickied', 'hidden', 'spoiler', 'over_18'],
|
|
})
|
|
.option('file-ignore', {
|
|
describe: 'Ignore the host IDs in this file',
|
|
type: 'string',
|
|
})
|
|
.option('include', {
|
|
describe: 'Include only these sources',
|
|
type: 'array',
|
|
})
|
|
.option('exclude', {
|
|
describe: 'Do not include these sources',
|
|
type: 'array',
|
|
})
|
|
.option('after', {
|
|
describe: 'Only include posts after this date (YYYY-MM-DD, optionally HH:mm)',
|
|
})
|
|
.option('before', {
|
|
describe: 'Only include posts before this date (YYYY-MM-DD, optionally HH:mm)',
|
|
})
|
|
.option('after-indexed', {
|
|
alias: 'after-index',
|
|
describe: 'Only include posts after the oldest or the latest entry in the index',
|
|
choices: ['oldest', 'latest'],
|
|
})
|
|
.option('before-indexed', {
|
|
alias: 'before-index',
|
|
describe: 'Only include posts before the oldest or the latest entry in the index',
|
|
choices: ['oldest', 'latest'],
|
|
})
|
|
.option('redownload', {
|
|
alias: 'force',
|
|
describe: 'Ignore index file and force a redownload of everything in the selection. Does not affect [before|after]-indexed',
|
|
type: 'boolean',
|
|
})
|
|
.option('redownload-profile', {
|
|
alias: 'force-profile',
|
|
describe: 'Ignore index file and force a redownload of the profile image and description',
|
|
type: 'boolean',
|
|
})
|
|
.option('index-ignored', {
|
|
describe: 'Add posts specified to be ignored to the index file, but don\'t download them',
|
|
type: 'boolean',
|
|
})
|
|
.option('watch', {
|
|
describe: 'Keep the process running and periodically check for new posts',
|
|
type: 'boolean',
|
|
})
|
|
.option('archives', {
|
|
describe: 'Search archives for deleted posts',
|
|
type: 'boolean',
|
|
default: config.fetch.archives.search,
|
|
})
|
|
.option('debug', {
|
|
describe: 'Log full stack traces instead of just error messages',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.argv;
|
|
|
|
return {
|
|
...args,
|
|
after: args.after ? new Date(args.after) : null,
|
|
before: args.before ? new Date(args.before) : null,
|
|
};
|
|
}
|
|
|
|
module.exports = getArgs;
|