357 lines
8.7 KiB
JavaScript
Executable File
357 lines
8.7 KiB
JavaScript
Executable File
'use strict';
|
|
|
|
const config = require('config');
|
|
const yargs = require('yargs');
|
|
const moment = require('moment');
|
|
|
|
function interpretAfter(after, ignoreIfEmpty = false) {
|
|
if (!after && ignoreIfEmpty) {
|
|
return null;
|
|
}
|
|
|
|
if (!after) {
|
|
return new Date(0, 0, 0);
|
|
}
|
|
|
|
if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) {
|
|
// using date
|
|
return moment
|
|
.utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
|
|
.toDate();
|
|
}
|
|
|
|
// using time distance (e.g. "1 month")
|
|
return moment
|
|
.utc()
|
|
.subtract(...after.split(' '))
|
|
.toDate();
|
|
}
|
|
|
|
const { argv } = yargs
|
|
.command('npm start')
|
|
.option('server', {
|
|
describe: 'Start web server',
|
|
type: 'boolean',
|
|
alias: 'web',
|
|
})
|
|
.option('include-networks', {
|
|
describe: 'Network to scrape all channels from (overrides configuration)',
|
|
type: 'array',
|
|
alias: ['include-network', 'networks', 'network'],
|
|
})
|
|
.option('exclude-networks', {
|
|
describe: 'Network not to scrape any channels from (overrides configuration)',
|
|
type: 'array',
|
|
alias: 'exclude-network',
|
|
})
|
|
.option('include-channels', {
|
|
describe: 'Channel to scrape (overrides configuration)',
|
|
type: 'array',
|
|
alias: ['include-channel', 'channels', 'channel'],
|
|
})
|
|
.option('exclude-channels', {
|
|
describe: 'Channel not to scrape (overrides configuration)',
|
|
type: 'array',
|
|
alias: 'exclude-channel',
|
|
})
|
|
.option('actors', {
|
|
describe: 'Scrape actors by name or slug',
|
|
type: 'array',
|
|
alias: 'actor',
|
|
})
|
|
.option('actors-update', {
|
|
describe: 'Rescrape actors last updated before this period',
|
|
type: 'string',
|
|
})
|
|
.option('actors-file', {
|
|
describe: 'Scrape actors names from file',
|
|
type: 'string',
|
|
})
|
|
.option('actors-scenes', {
|
|
describe: 'Fetch all scenes for an actor',
|
|
type: 'boolean',
|
|
default: false,
|
|
alias: 'actor-scenes',
|
|
})
|
|
.option('actor-sources', {
|
|
describe: 'Use these scrapers for actor data',
|
|
type: 'array',
|
|
alias: ['actor-source', 'profile-sources', 'profile-source', 'source', 'sources'],
|
|
})
|
|
.option('movie-scenes', {
|
|
describe: 'Fetch all scenes for a movie',
|
|
type: 'boolean',
|
|
alias: 'with-scenes',
|
|
default: false,
|
|
})
|
|
.option('scene-movies', {
|
|
describe: 'Fetch movies for scenes',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('scene-actors', {
|
|
describe: 'Scrape profiles for new actors after fetching scenes',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('scene', {
|
|
describe: 'Scrape scene info from URL',
|
|
type: 'array',
|
|
})
|
|
.option('scene-file', {
|
|
describe: 'Scrape scene info from URLs in a file',
|
|
type: 'string',
|
|
alias: 'scenes-file',
|
|
})
|
|
.option('movie', {
|
|
describe: 'Scrape movie info from URL',
|
|
type: 'array',
|
|
})
|
|
.option('movie-file', {
|
|
describe: 'Scrape movie info from URLs in a file',
|
|
type: 'string',
|
|
alias: 'movies-file',
|
|
})
|
|
.option('deep', {
|
|
describe: 'Fetch details for all releases',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('latest', {
|
|
describe: 'Scrape latest releases if available',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('upcoming', {
|
|
describe: 'Scrape upcoming releases if available',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('paginate-upcoming', {
|
|
describe: 'Attempt \'next\' upcoming page, usually unavailable.',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('movies', {
|
|
describe: 'Scrape movies from channels',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('force', {
|
|
describe: 'Don\'t ignore duplicates, update existing entries',
|
|
type: 'boolean',
|
|
alias: 'redownload',
|
|
})
|
|
.option('after', {
|
|
describe: 'Don\'t fetch scenes older than',
|
|
type: 'string',
|
|
default: config.fetchAfter.join(' '),
|
|
})
|
|
.option('last', {
|
|
describe: 'Get the latest x releases, no matter the date range',
|
|
type: 'number',
|
|
})
|
|
.option('missing-date-limit', {
|
|
describe: 'Limit amount of scenes when dates are missing.',
|
|
type: 'number',
|
|
default: config.missingDateLimit,
|
|
alias: ['null-date-limit'],
|
|
})
|
|
.option('page', {
|
|
describe: 'Page to start scraping at',
|
|
type: 'number',
|
|
default: 1,
|
|
})
|
|
.option('interval', {
|
|
describe: 'Minimum wait time between HTTP requests',
|
|
type: 'number',
|
|
// don't set default, because argument has to override config, but config has to override default
|
|
})
|
|
.option('concurrency', {
|
|
describe: 'Maximum amount of parallel HTTP requests',
|
|
type: 'number',
|
|
// don't set default, because argument has to override config, but config has to override default
|
|
})
|
|
.option('save', {
|
|
describe: 'Save fetched releases to database',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('media', {
|
|
describe: 'Include any release media',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('media-limit', {
|
|
describe: 'Maximum amount of assets of each type per release',
|
|
type: 'number',
|
|
default: config.media.limit,
|
|
})
|
|
.option('images', {
|
|
describe: 'Include any photos, posters or covers',
|
|
type: 'boolean',
|
|
default: true,
|
|
alias: 'pics',
|
|
})
|
|
.option('videos', {
|
|
alias: 'video',
|
|
describe: 'Include any trailers or teasers',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('posters', {
|
|
describe: 'Include release posters',
|
|
type: 'boolean',
|
|
default: true,
|
|
alias: 'poster',
|
|
})
|
|
.option('covers', {
|
|
describe: 'Include release covers',
|
|
type: 'boolean',
|
|
default: true,
|
|
alias: 'cover',
|
|
})
|
|
.option('photos', {
|
|
describe: 'Include release photos',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('trailers', {
|
|
describe: 'Include release trailers',
|
|
type: 'boolean',
|
|
default: true,
|
|
alias: 'trailer',
|
|
})
|
|
.option('teasers', {
|
|
describe: 'Include release teasers',
|
|
type: 'boolean',
|
|
default: true,
|
|
alias: 'teaser',
|
|
})
|
|
.option('avatars', {
|
|
describe: 'Include actor avatars',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('report', {
|
|
describe: 'Show data in console.',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('level', {
|
|
alias: 'log-level',
|
|
describe: 'Log level',
|
|
type: 'string',
|
|
default: process.env.NODE_ENV === 'development' ? 'silly' : 'info',
|
|
})
|
|
.option('resolve-place', {
|
|
describe: 'Call OSM Nominatim API for actor place of birth and residence. Raw value discarded if disabled.',
|
|
type: 'boolean',
|
|
default: true,
|
|
})
|
|
.option('debug', {
|
|
describe: 'Show error stack traces',
|
|
type: 'boolean',
|
|
default: process.env.NODE_ENV === 'development',
|
|
})
|
|
.option('sampleMemory', {
|
|
alias: 'memory',
|
|
describe: 'Take memory allocation samples, and snapshots at configured intervals',
|
|
type: 'boolean',
|
|
default: config.memorySampling.enabled,
|
|
})
|
|
.option('update-search', {
|
|
describe: 'Update search documents for all releases.',
|
|
type: 'boolean',
|
|
default: false,
|
|
})
|
|
.option('interpolate-profiles', {
|
|
describe: 'Interpolate actor profiles',
|
|
type: 'array',
|
|
alias: 'interpolate',
|
|
})
|
|
.option('flush-orphaned-media', {
|
|
describe: 'Remove all orphaned media items from database and disk.',
|
|
type: 'boolean',
|
|
alias: 'flush-media',
|
|
})
|
|
.option('flush-media-files', {
|
|
describe: 'Remove files from storage when flushing media.',
|
|
type: 'boolean',
|
|
alias: 'flush-files',
|
|
default: true,
|
|
})
|
|
.option('flush-channels', {
|
|
describe: 'Delete all scenes and movies from channels.',
|
|
type: 'array',
|
|
alias: 'flush-channel',
|
|
})
|
|
.option('flush-networks', {
|
|
describe: 'Delete all scenes and movies from network.',
|
|
type: 'array',
|
|
alias: 'flush-network',
|
|
})
|
|
.option('flush-actors', {
|
|
describe: 'Flush all actors and their scenes.',
|
|
type: 'array',
|
|
alias: 'flush-actor',
|
|
})
|
|
.option('flush-profiles', {
|
|
describe: 'Delete all profiles for an actor.',
|
|
type: 'array',
|
|
alias: 'flush-profile',
|
|
})
|
|
.option('flush-batches', {
|
|
describe: 'Delete all scenes and movies from batch by ID.',
|
|
type: 'array',
|
|
alias: 'flush-batch',
|
|
})
|
|
.option('flush-scenes', {
|
|
describe: 'Remove all scenes.',
|
|
type: 'boolean',
|
|
})
|
|
.option('flush-movies', {
|
|
describe: 'Remove all movies.',
|
|
type: 'boolean',
|
|
})
|
|
.option('delete-actors', {
|
|
describe: 'Remove actors by ID.',
|
|
type: 'array',
|
|
alias: ['delete-actor', 'remove-actors', 'remove-actor'],
|
|
})
|
|
.option('delete-scenes', {
|
|
describe: 'Remove scenes by ID.',
|
|
type: 'array',
|
|
alias: ['delete-scene', 'delete', 'remove', 'remove-scenes', 'remove-scene'],
|
|
})
|
|
.option('delete-movies', {
|
|
describe: 'Remove movies by ID.',
|
|
type: 'array',
|
|
alias: ['delete-movie', 'remove-movies', 'remove-movies'],
|
|
})
|
|
.option('request', {
|
|
describe: 'Make an arbitrary HTTP request',
|
|
type: 'string',
|
|
})
|
|
.option('request-method', {
|
|
alias: 'method',
|
|
describe: 'HTTP method for arbitrary HTTP requests',
|
|
type: 'string',
|
|
default: 'get',
|
|
})
|
|
.option('request-timeout', {
|
|
describe: 'Default timeout after which to cancel a HTTP request.',
|
|
type: 'number',
|
|
alias: ['timeout'],
|
|
default: 60000,
|
|
})
|
|
.option('media-source', {
|
|
describe: 'Traxxx host to use for media transfers.',
|
|
type: 'string',
|
|
default: config.media.transferSource,
|
|
})
|
|
.coerce('after', interpretAfter)
|
|
.coerce('actors-update', (after) => interpretAfter(after, true));
|
|
|
|
module.exports = argv;
|