Added basic actor and network overview. Added DDF Network actor scraper. Various bug fixes and layout improvements.

src/actors.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const Promise = require('bluebird');
+const UrlPattern = require('url-pattern');
 
 const knex = require('./knex');
 const argv = require('./argv');
@@ -54,6 +55,7 @@ async function curateActor(actor) {
         curatedActor.origin.country = {
             alpha2: actor.birth_country_alpha2,
             name: actor.birth_country_name,
+            alias: actor.birth_country_alias,
         };
     }
 
@@ -64,6 +66,7 @@ async function curateActor(actor) {
         curatedActor.residence.country = {
             alpha2: actor.residence_country_alpha2,
             name: actor.residence_country_name,
+            alias: actor.residence_country_alias,
         };
     }
 
@@ -124,12 +127,54 @@ function curateActorEntry(actor, scraped, scrapeSuccess) {
 }
 
 function curateSocialEntry(url, actorId) {
-    const { hostname, origin, pathname } = new URL(url);
-    const platform = ['facebook', 'twitter', 'instagram', 'tumblr', 'snapchat', 'amazon', 'youtube', 'fancentro'].find(platformName => hostname.match(platformName));
+    const platforms = [
+        {
+            label: 'twitter',
+            pattern: 'http(s)\\://(*)twitter.com/:username(/)(?*)',
+            format: username => `https://www.twitter.com/${username}`,
+        },
+        {
+            label: 'instagram',
+            pattern: 'http(s)\\://(*)instagram.com/:username(/)(?*)',
+            format: username => `https://www.instagram.com/${username}`,
+        },
+        {
+            label: 'snapchat',
+            pattern: 'http(s)\\://(*)snapchat.com/add/:username(/)(?*)',
+            format: username => `https://www.snapchat.com/add/${username}`,
+        },
+        {
+            label: 'tumblr',
+            pattern: 'http(s)\\://:username.tumblr.com(*)',
+            format: username => `https://${username}.tumblr.com`,
+        },
+        {
+            label: 'fancentro',
+            pattern: 'http(s)\\://(www.)fancentro.com/:username(/)(?*)',
+            format: username => `https://www.fancentro.com/${username}`,
+        },
+    ];
+
+    const match = platforms.reduce((acc, platform) => {
+        if (acc) return acc;
+
+        const patternMatch = new UrlPattern(platform.pattern).match(url);
+
+        if (patternMatch) {
+            return {
+                platform: platform.label,
+                original: url,
+                username: patternMatch.username,
+                url: platform.format ? platform.format(patternMatch.username) : url,
+            };
+        }
+
+        return null;
+    }, null) || { url };
 
     return {
-        url: `${origin}${pathname}`,
-        platform,
+        url: match.url,
+        platform: match.platform,
         domain: 'actors',
         target_id: actorId,
     };
@@ -148,7 +193,7 @@ async function curateSocialEntries(urls, actorId) {
     return urls.reduce((acc, url) => {
         const socialEntry = curateSocialEntry(url, actorId);
 
-        if (acc.some(entry => socialEntry.url === entry.url) || existingSocialLinks.some(entry => socialEntry.url === entry.url)) {
+        if (acc.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase()) || existingSocialLinks.some(entry => socialEntry.url.toLowerCase() === entry.url.toLowerCase())) {
             // prevent duplicates
             return acc;
         }
@@ -161,11 +206,12 @@ async function fetchActors(queryObject) {
     const releases = await knex('actors')
         .select(
             'actors.*',
-            'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name',
-            'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name',
+            'birth_countries.alpha2 as birth_country_alpha2', 'birth_countries.name as birth_country_name', 'birth_countries.alias as birth_country_alias',
+            'residence_countries.alpha2 as residence_country_alpha2', 'residence_countries.name as residence_country_name', 'residence_countries.alias as residence_country_alias',
         )
         .leftJoin('countries as birth_countries', 'actors.birth_country_alpha2', 'birth_countries.alpha2')
         .leftJoin('countries as residence_countries', 'actors.residence_country_alpha2', 'residence_countries.alpha2')
+        .orderBy(['actors.name', 'actors.gender'])
         .where(builder => whereOr(queryObject, 'actors', builder))
         .limit(100);
 
@@ -215,7 +261,7 @@ async function mergeProfiles(profiles, actor) {
 
         return {
             id: actor ? actor.id : null,
-            name: actor ? actor.name : profile.name,
+            name: actor ? actor.name : (prevProfile.name || profile.name),
             description: prevProfile.description || profile.description,
             gender: prevProfile.gender || profile.gender,
             birthdate: Number.isNaN(Number(prevProfile.birthdate)) ? profile.birthdate : prevProfile.birthdate,
@@ -257,9 +303,10 @@ async function scrapeActors(actorNames) {
     await Promise.map(actorNames || argv.actors, async (actorName) => {
         try {
             const actorSlug = actorName.toLowerCase().replace(/\s+/g, '-');
-
             const actorEntry = await knex('actors').where({ slug: actorSlug }).first();
-            const profiles = await Promise.map(Object.entries(scrapers.actors), async ([scraperSlug, scraper]) => {
+            const sources = argv.sources ? argv.sources.map(source => [source, scrapers.actors[source]]) : Object.entries(scrapers.actors);
+
+            const profiles = await Promise.map(sources, async ([scraperSlug, scraper]) => {
                 const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName);
 
                 return {