DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 40 Pull Requests 1 Releases Wiki Activity

Redundant actor sources can now be bundled in configuration. Fixed Men network actor path.

Browse Source
This commit is contained in:
ThePendulum
2020-02-01 04:14:08 +01:00
parent 5ff916475a
commit cde9aba0cb
4 changed files with 37 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/actors.js
View File

@@ -1,6 +1,6 @@
'use strict';
// const config = require('config');
const config = require('config');
const Promise = require('bluebird');
const UrlPattern = require('url-pattern');
const moment = require('moment');
@@ -339,21 +339,36 @@ async function scrapeActors(actorNames) {
try {
const actorSlug = slugify(actorName);
const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
const sources = argv.sources ? argv.sources.map(source => [source, scrapers.actors[source]]) : Object.entries(scrapers.actors);
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const profiles = await Promise.map(sources, async (source) => {
// const [scraperSlug, scraper] = source;
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
const profiles = await Promise.map(sources, async ([scraperSlug, scraper]) => {
try {
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
return profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
return {
...profile,
name: actorName,
scraper: scraperSlug,
};
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
if (profile) {
logger.verbose(`Found profile for '${actorName}' on ${scraperSlug}`);
return {
...profile,
name: actorName,
scraper: scraperSlug,
};
}
logger.verbose(`No profile for '${actorName}' available on ${scraperSlug}`);
throw new Error(`Profile for ${actorName} not available on ${scraperSlug}`);
}), Promise.reject(new Error()));
} catch (error) {
console.log(scraperSlug, error);
return null;
logger.error(error);
}
return null;
});
const profile = await mergeProfiles(profiles, actorEntry);