Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

This commit is contained in:
2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions

View File

@@ -1,5 +1,6 @@
'use strict';
const config = require('config');
const yargs = require('yargs');
const { argv } = yargs
@@ -14,6 +15,16 @@ const { argv } = yargs
type: 'array',
alias: 'site',
})
.option('after', {
describe: 'Don\'t fetch scenes older than',
type: 'string',
default: config.fetchAfter.join(' '),
})
.option('save', {
describe: 'Save fetched releases to database',
type: 'boolean',
default: true,
})
.option('render', {
describe: 'Fetch data without rendering interface',
type: 'boolean',