boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Added profile scraper with releases to Hush. Added qtexts to q to return text nodes individually. Including network in profile site.

Browse Source
This commit is contained in:
ThePendulum
2020-03-06 02:49:55 +01:00
parent ae5bd374ba
commit db01599569
36 changed files with 298 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/app.js
View File

@@ -5,7 +5,8 @@ const knex = require('./knex');
const initServer = require('./web/server');
const scrapeSites = require('./scrape-sites');
const { scrapeReleases } = require('./scrape-releases');
const { scrapeReleases, deepFetchReleases } = require('./scrape-releases');
const { storeReleases } = require('./releases');
const { scrapeActors, scrapeBasicActors } = require('./actors');
if (process.env.NODE_ENV === 'development') {
@@ -29,9 +30,10 @@ async function init() {
const actors = await scrapeActors();
if (argv.withReleases) {
const releases = actors.map(actor => actor?.releases || []).flat();
const baseReleases = actors.map(actor => actor?.releases || []).flat();
const releases = await deepFetchReleases(baseReleases, null, 'scene');
await scrapeReleases(releases, null, 'scene');
await storeReleases(releases);
}
}