boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding networks and sites as entities,

This commit is contained in:
ThePendulum 2020-06-04 01:03:02 +02:00
parent 8abcc7194a
commit 09d849eb9d
9 changed files with 283 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/components/tile/tag.vue
View File

@ -1,6 +1,6 @@
<template> <template>
<router-link <router-link
:to="{ name: 'tag', params: { tagSlug: tag.slug } }" :to="{ name: 'tag', params: { tagSlug: tag.slug, tags: 'all-tags', range: 'latest' } }"
:title="tag.name" :title="tag.name"
class="tile" class="tile"
> >

27
config/default.js
View File

@ -14,8 +14,7 @@ module.exports = {
// include: [], // include: [],
// exclude: [], // exclude: [],
exclude: [ exclude: [
['21sextreme', [ // 21sextreme, no longer updated
// no longer updated
'mightymistress', 'mightymistress',
'dominatedgirls', 'dominatedgirls',
'homepornreality', 'homepornreality',
@ -24,24 +23,23 @@ module.exports = {
'mandyiskinky', 'mandyiskinky',
'speculumplays', 'speculumplays',
'creampiereality', 'creampiereality',
]], // aziani
['aziani', [
'amberathome', 'amberathome',
'marycarey', 'marycarey',
'racqueldevonshire', 'racqueldevonshire',
]], // boobpedia
'boobpedia', 'boobpedia',
['blowpass', ['sunlustxxx']], // blowpass
['ddfnetwork', [ 'sunlustxxx',
// ddfnetwork
'fuckinhd', 'fuckinhd',
'bustylover', 'bustylover',
]], // famedigital
['famedigital', [
'daringsex', 'daringsex',
'lowartfilms', 'lowartfilms',
]], // freeones
'freeones', 'freeones',
['pornpros', [ // pornpros
'milfhumiliation', 'milfhumiliation',
'humiliated', 'humiliated',
'flexiblepositions', 'flexiblepositions',
@ -51,17 +49,14 @@ module.exports = {
'cumdisgrace', 'cumdisgrace',
'webcamhackers', 'webcamhackers',
'collegeteens', 'collegeteens',
]], // score
['score', [
'bigboobbundle', 'bigboobbundle',
'milfbundle', 'milfbundle',
'pornmegaload', 'pornmegaload',
'scorelandtv', 'scorelandtv',
'scoretv', 'scoretv',
]], // mindgeek
['mindgeek', [
'pornhub', 'pornhub',
]],
], ],
profiles: [ profiles: [
[ [

57
migrations/20190325001339_releases.js
View File

@ -140,6 +140,59 @@ exports.up = knex => Promise.resolve()
table.unique(['tag_id', 'media_id']); table.unique(['tag_id', 'media_id']);
})) }))
.then(() => knex.schema.createTable('entities_types', (table) => {
table.increments('id', 4);
table.text('type');
}))
.then(() => knex('entities_types').insert([
{ type: 'network' },
{ type: 'channel' },
{ type: 'studio' },
]))
.then(() => knex.schema.createTable('entities', (table) => {
table.increments('id', 12);
table.integer('parent_id', 12)
.references('id')
.inTable('entities');
table.integer('type', 4)
.references('id')
.inTable('entities_types')
.defaultTo(2);
table.text('name');
table.text('slug', 32)
.unique();
table.text('alias');
table.text('url');
table.text('description');
table.json('parameters');
table.boolean('active');
table.integer('priority', 3)
.defaultTo(0);
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('entities_tags', (table) => {
table.integer('tag_id', 12)
.notNullable()
.references('id')
.inTable('tags');
table.integer('entity_id', 12)
.notNullable()
.references('id')
.inTable('entities');
table.boolean('inherit')
.defaultTo(false);
table.unique(['tag_id', 'entity_id']);
}))
.then(() => knex.schema.createTable('networks', (table) => { .then(() => knex.schema.createTable('networks', (table) => {
table.increments('id', 12); table.increments('id', 12);
@ -897,6 +950,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS actors_piercings CASCADE; DROP TABLE IF EXISTS actors_piercings CASCADE;
DROP TABLE IF EXISTS body CASCADE; DROP TABLE IF EXISTS body CASCADE;
DROP TABLE IF EXISTS entities_tags CASCADE;
DROP TABLE IF EXISTS sites_tags CASCADE; DROP TABLE IF EXISTS sites_tags CASCADE;
DROP TABLE IF EXISTS sites_social CASCADE; DROP TABLE IF EXISTS sites_social CASCADE;
DROP TABLE IF EXISTS networks_social CASCADE; DROP TABLE IF EXISTS networks_social CASCADE;
@ -915,6 +969,9 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS countries CASCADE; DROP TABLE IF EXISTS countries CASCADE;
DROP TABLE IF EXISTS networks CASCADE; DROP TABLE IF EXISTS networks CASCADE;
DROP TABLE IF EXISTS entities_types CASCADE;
DROP TABLE IF EXISTS entities CASCADE;
DROP FUNCTION IF EXISTS search_sites; DROP FUNCTION IF EXISTS search_sites;
DROP FUNCTION IF EXISTS search_actors; DROP FUNCTION IF EXISTS search_actors;
DROP FUNCTION IF EXISTS get_random_sfw_media_id; DROP FUNCTION IF EXISTS get_random_sfw_media_id;

4
seeds/01_networks.js
View File

@ -430,7 +430,7 @@ const networks = [
exports.seed = knex => Promise.resolve() exports.seed = knex => Promise.resolve()
.then(async () => { .then(async () => {
const { inserted, updated } = await upsert('networks', parentNetworks, 'slug', knex); const { inserted, updated } = await upsert('entities', parentNetworks, 'slug', knex);
const parentNetworksBySlug = [].concat(inserted, updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {}); const parentNetworksBySlug = [].concat(inserted, updated).reduce((acc, network) => ({ ...acc, [network.slug]: network.id }), {});
const networksWithParent = networks.map(network => ({ const networksWithParent = networks.map(network => ({
@ -443,5 +443,5 @@ exports.seed = knex => Promise.resolve()
parent_id: parentNetworksBySlug[network.parent] || null, parent_id: parentNetworksBySlug[network.parent] || null,
})); }));
return upsert('networks', networksWithParent, 'slug', knex); return upsert('entities', networksWithParent, 'slug', knex);
}); });

27
seeds/02_sites.js
View File

@ -1158,7 +1158,7 @@ const sites = [
url: 'https://www.sunlustxxx.com', url: 'https://www.sunlustxxx.com',
description: '', description: '',
network: 'blowpass', network: 'blowpass',
show: true, // site offline, use only for indexing old scenes active: true, // site offline, use only for indexing old scenes
}, },
// BOOBPEDIA // BOOBPEDIA
{ {
@ -1877,7 +1877,7 @@ const sites = [
description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.', description: 'Welcome the official Daring Sex site, home of high quality erotica, sensual porn and hardcore exploration of the darker side of sexuality. Here you will find a variety of videos for lovers looking for a bit of extra, or something darker with an element of control.',
network: 'famedigital', network: 'famedigital',
parameters: { api: true }, parameters: { api: true },
show: false, // no data sources active: false, // no data sources
}, },
{ {
slug: 'peternorth', slug: 'peternorth',
@ -4157,6 +4157,7 @@ const sites = [
{ {
name: 'Teen BFF', name: 'Teen BFF',
slug: 'teenbff', slug: 'teenbff',
alias: ['tbff'],
url: 'https://pornpros.com/site/teenbff', url: 'https://pornpros.com/site/teenbff',
tags: ['mff'], tags: ['mff'],
network: 'pornpros', network: 'pornpros',
@ -4921,7 +4922,7 @@ const sites = [
slug: 'bigboobbundle', slug: 'bigboobbundle',
url: 'https://www.bigboobbundle.com', url: 'https://www.bigboobbundle.com',
network: 'score', network: 'score',
show: false, // all content appears to be on subsites active: false, // all content appears to be on subsites
}, },
{ {
name: 'Big Boobs POV', name: 'Big Boobs POV',
@ -5213,7 +5214,7 @@ const sites = [
slug: 'milfbundle', slug: 'milfbundle',
url: 'https://www.milfbundle.com', url: 'https://www.milfbundle.com',
network: 'score', network: 'score',
show: false, active: false,
}, },
{ {
name: 'Teaming Cock', name: 'Teaming Cock',
@ -5280,7 +5281,7 @@ const sites = [
slug: 'pornmegaload', slug: 'pornmegaload',
url: 'https://www.pornmegaload.com', url: 'https://www.pornmegaload.com',
network: 'score', network: 'score',
show: false, active: false,
}, },
{ {
name: 'SaRennas World', name: 'SaRennas World',
@ -5318,7 +5319,7 @@ const sites = [
url: 'https://www.scorepass.com/scorelandtv', url: 'https://www.scorepass.com/scorelandtv',
network: 'score', network: 'score',
priority: 1, priority: 1,
show: false, // appears to be streaming service for other sites active: false, // appears to be streaming service for other sites
}, },
{ {
name: 'ScoreTV', name: 'ScoreTV',
@ -5326,7 +5327,7 @@ const sites = [
url: 'https://www.scoretv.tv', url: 'https://www.scoretv.tv',
network: 'score', network: 'score',
priority: 1, priority: 1,
show: false, // similar to or same as Scoreland TV active: false, // similar to or same as Scoreland TV
}, },
{ {
name: 'Score Videos', name: 'Score Videos',
@ -6177,7 +6178,7 @@ const sites = [
/* eslint-disable max-len */ /* eslint-disable max-len */
exports.seed = knex => Promise.resolve() exports.seed = knex => Promise.resolve()
.then(async () => { .then(async () => {
const networks = await knex('networks').select('*'); const networks = await knex('entities').select('*');
const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tags = await knex('tags').select('*').where('alias_for', null); const tags = await knex('tags').select('*').where('alias_for', null);
@ -6190,24 +6191,24 @@ exports.seed = knex => Promise.resolve()
description: site.description, description: site.description,
url: site.url, url: site.url,
parameters: site.parameters, parameters: site.parameters,
network_id: networksMap[site.network], parent_id: networksMap[site.network],
priority: site.priority, priority: site.priority,
show: site.show, active: site.show,
})); }));
const { inserted, updated } = await upsert('sites', sitesWithNetworks, 'slug', knex); const { inserted, updated } = await upsert('entities', sitesWithNetworks, 'slug', knex);
const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); const sitesMap = [].concat(inserted, updated).reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const tagAssociations = sites.map(site => (site.tags const tagAssociations = sites.map(site => (site.tags
? site.tags.map(tagSlug => ({ ? site.tags.map(tagSlug => ({
site_id: sitesMap[site.slug], entity_id: sitesMap[site.slug],
tag_id: tagsMap[tagSlug], tag_id: tagsMap[tagSlug],
inherit: true, inherit: true,
})) }))
: [] : []
)).flat(); )).flat();
return upsert('sites_tags', tagAssociations, ['site_id', 'tag_id'], knex); return upsert('entities_tags', tagAssociations, ['entity_id', 'tag_id'], knex);
}); });
/* /*

62
seeds/03_studios.js
View File

@ -7,145 +7,145 @@ function getStudios(networksMap) {
slug: 'gonzocom', slug: 'gonzocom',
name: 'Gonzo.com', name: 'Gonzo.com',
url: 'https://www.legalporno.com/studios/gonzo_com', url: 'https://www.legalporno.com/studios/gonzo_com',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'giorgiograndi', slug: 'giorgiograndi',
name: 'Giorgio Grandi', name: 'Giorgio Grandi',
url: 'https://www.legalporno.com/studios/giorgio-grandi', url: 'https://www.legalporno.com/studios/giorgio-grandi',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'hardpornworld', slug: 'hardpornworld',
name: 'Hard Porn World', name: 'Hard Porn World',
url: 'https://www.legalporno.com/studios/hard-porn-world', url: 'https://www.legalporno.com/studios/hard-porn-world',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'interracialvision', slug: 'interracialvision',
name: 'Interracial Vision', name: 'Interracial Vision',
url: 'https://www.legalporno.com/studios/interracial-vision', url: 'https://www.legalporno.com/studios/interracial-vision',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'giorgioslab', slug: 'giorgioslab',
name: 'Giorgio\'s Lab', name: 'Giorgio\'s Lab',
url: 'https://www.legalporno.com/studios/giorgio--s-lab', url: 'https://www.legalporno.com/studios/giorgio--s-lab',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'americananal', slug: 'americananal',
name: 'American Anal', name: 'American Anal',
url: 'https://www.legalporno.com/studios/american-anal', url: 'https://www.legalporno.com/studios/american-anal',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'assablanca', slug: 'assablanca',
name: 'Assablanca', name: 'Assablanca',
url: 'https://www.legalporno.com/studios/assablanca', url: 'https://www.legalporno.com/studios/assablanca',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'focus', slug: 'focus',
name: 'Focus', name: 'Focus',
url: 'https://www.legalporno.com/studios/focus', url: 'https://www.legalporno.com/studios/focus',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'analforever', slug: 'analforever',
name: 'Anal Forever', name: 'Anal Forever',
url: 'https://www.legalporno.com/studios/anal-forever', url: 'https://www.legalporno.com/studios/anal-forever',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'gonzoinbrazil', slug: 'gonzoinbrazil',
name: 'Gonzo in Brazil', name: 'Gonzo in Brazil',
url: 'https://www.legalporno.com/studios/gonzo-in-brazil', url: 'https://www.legalporno.com/studios/gonzo-in-brazil',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'mranal', slug: 'mranal',
name: 'Mr Anal', name: 'Mr Anal',
url: 'https://www.legalporno.com/studios/mr-anal', url: 'https://www.legalporno.com/studios/mr-anal',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'tarrawhite', slug: 'tarrawhite',
name: 'Tarra White', name: 'Tarra White',
url: 'https://www.legalporno.com/studios/tarra-white', url: 'https://www.legalporno.com/studios/tarra-white',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'sineplexsos', slug: 'sineplexsos',
name: 'Sineplex SOS', name: 'Sineplex SOS',
url: 'https://www.legalporno.com/studios/sineplex-sos', url: 'https://www.legalporno.com/studios/sineplex-sos',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'fmodels', slug: 'fmodels',
name: 'F Models', name: 'F Models',
url: 'https://www.legalporno.com/studios/f-models', url: 'https://www.legalporno.com/studios/f-models',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'sineplexcz', slug: 'sineplexcz',
name: 'Sineplex CZ', name: 'Sineplex CZ',
url: 'https://www.legalporno.com/studios/sineplex-cz', url: 'https://www.legalporno.com/studios/sineplex-cz',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'gg', slug: 'gg',
name: 'GG', name: 'GG',
url: 'https://www.legalporno.com/studios/gg', url: 'https://www.legalporno.com/studios/gg',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'firstgape', slug: 'firstgape',
name: 'First Gape', name: 'First Gape',
url: 'https://www.legalporno.com/studios/first-gape', url: 'https://www.legalporno.com/studios/first-gape',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'omargalantiproductions', slug: 'omargalantiproductions',
name: 'Omar Galanti Productions', name: 'Omar Galanti Productions',
url: 'https://www.legalporno.com/studios/omar-galanti-productions', url: 'https://www.legalporno.com/studios/omar-galanti-productions',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'norestfortheass', slug: 'norestfortheass',
name: 'No Rest For The Ass', name: 'No Rest For The Ass',
url: 'https://www.legalporno.com/studios/no-rest-for-the-ass', url: 'https://www.legalporno.com/studios/no-rest-for-the-ass',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'hairygonzo', slug: 'hairygonzo',
name: 'Hairy Gonzo', name: 'Hairy Gonzo',
url: 'https://www.legalporno.com/studios/hairy-gonzo', url: 'https://www.legalporno.com/studios/hairy-gonzo',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'sineplexclassic', slug: 'sineplexclassic',
name: 'Sineplex Classic', name: 'Sineplex Classic',
url: 'https://www.legalporno.com/studios/sineplex-classic', url: 'https://www.legalporno.com/studios/sineplex-classic',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'sinemale', slug: 'sinemale',
name: 'Sinemale', name: 'Sinemale',
url: 'https://www.legalporno.com/studios/sinemale', url: 'https://www.legalporno.com/studios/sinemale',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'outsidethestudio', slug: 'outsidethestudio',
name: 'Outside The Studio', name: 'Outside The Studio',
url: 'https://www.legalporno.com/studios/outside-the-studio', url: 'https://www.legalporno.com/studios/outside-the-studio',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
{ {
slug: 'kinkysex', slug: 'kinkysex',
name: 'Kinky Sex', name: 'Kinky Sex',
url: 'https://www.legalporno.com/studios/kinky-sex', url: 'https://www.legalporno.com/studios/kinky-sex',
network_id: networksMap.legalporno, parent_id: networksMap.legalporno,
}, },
]; ];
} }
@ -153,10 +153,18 @@ function getStudios(networksMap) {
/* eslint-disable max-len */ /* eslint-disable max-len */
exports.seed = knex => Promise.resolve() exports.seed = knex => Promise.resolve()
.then(async () => { .then(async () => {
const networks = await knex('networks').select('*'); const [networks, types] = await Promise.all([
knex('entities').select('*'),
knex('entities_types').select('*'),
]);
const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {}); const networksMap = networks.reduce((acc, { id, slug }) => ({ ...acc, [slug]: id }), {});
const typesMap = types.reduce((acc, type) => ({ [type.type]: type.id }), {});
const studios = getStudios(networksMap); const studios = getStudios(networksMap).map(studio => ({
...studio,
type: typesMap.studio,
}));
return upsert('studios', studios, 'slug', knex); return upsert('entities', studios, 'slug', knex);
}); });

3
seeds/04_media.js
View File

@ -591,7 +591,7 @@ const tagPosters = [
['anal-creampie', 1, 'Aleska Diamond in "Aleska Wants More" for Asshole Fever'], ['anal-creampie', 1, 'Aleska Diamond in "Aleska Wants More" for Asshole Fever'],
['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'], ['ass-eating', 0, 'Angelica Heart and Leanna Sweet in "ATM Bitches" for Asshole Fever'],
['asian', 0, 'Alina Li in "Slut Puppies 8" for Jules Jordan'], ['asian', 0, 'Alina Li in "Slut Puppies 8" for Jules Jordan'],
['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'], ['atm', 6, 'Jane Wilde in "Teen Anal" for Evil Angel'],
['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'], ['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'], ['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'], ['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
@ -659,6 +659,7 @@ const tagPhotos = [
['airtight', 2, 'Dakota Skye in "Dakota Goes Nuts" for ArchAngel'], ['airtight', 2, 'Dakota Skye in "Dakota Goes Nuts" for ArchAngel'],
['airtight', 3, 'Anita Bellini in "Triple Dick Gangbang" for Hands On Hardcore (DDF Network)'], ['airtight', 3, 'Anita Bellini in "Triple Dick Gangbang" for Hands On Hardcore (DDF Network)'],
['anal-creampie', 0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'], ['anal-creampie', 0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
['atm', 2, 'Jureka Del Mar in "Stretched Out" for Her Limit'],
['atm', 0, 'Roxy Lips in "Under Her Coat" for 21 Naturals'], ['atm', 0, 'Roxy Lips in "Under Her Coat" for 21 Naturals'],
['atm', 3, 'Natasha Teen in "Work That Ass!" for Her Limit'], ['atm', 3, 'Natasha Teen in "Work That Ass!" for Her Limit'],
['asian', 'poster', 'Vina Sky in "Slut Puppies 15" for Jules Jordan'], ['asian', 'poster', 'Vina Sky in "Slut Puppies 15" for Jules Jordan'],

2
src/argv.js
View File

@ -22,7 +22,7 @@ const { argv } = yargs
.option('sites', { .option('sites', {
describe: 'Sites to scrape (overrides configuration)', describe: 'Sites to scrape (overrides configuration)',
type: 'array', type: 'array',
alias: 'site', alias: 'entities',
}) })
.option('actors', { .option('actors', {
describe: 'Scrape actors by name or slug', describe: 'Scrape actors by name or slug',

8
src/updates.js
View File

@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const include = require('./utils/argv-include')(argv); const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers'); const scrapers = require('./scrapers/scrapers');
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./sites'); const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');
const afterDate = (() => { const afterDate = (() => {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) { if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
@ -218,9 +218,9 @@ async function scrapeNetworkParallel(network) {
} }
async function fetchUpdates() { async function fetchUpdates() {
const includedNetworks = argv.sites || argv.networks const includedNetworks = argv.sites || argv.networks || argv.from
? await fetchSitesFromArgv() ? await fetchEntitiesFromArgv()
: await fetchSitesFromConfig(); : await fetchEntitiesFromConfig();
const scrapedNetworks = await Promise.map( const scrapedNetworks = await Promise.map(
includedNetworks, includedNetworks,