Added source maps. Fixed DDFNetwork avatar and improved photo fetch error handling.

This commit is contained in:
ThePendulum 2019-12-10 00:30:55 +01:00
parent 04a89efa58
commit b6123b9418
9 changed files with 17 additions and 16 deletions

View File

@ -4,5 +4,5 @@
"@babel/preset-env" "@babel/preset-env"
] ]
], ],
"plugins": ["@babel/plugin-proposal-optional-chaining"] "plugins": ["@babel/plugin-proposal-optional-chaining"],
} }

View File

@ -9,7 +9,8 @@
"rules": { "rules": {
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0, "no-console": 0,
"indent": ["error", 4], "indent": "off",
"template-curly-spacing": "off",
"max-len": [2, { "max-len": [2, {
"code": 300, "code": 300,
"tabWidth": 4, "tabWidth": 4,
@ -21,7 +22,7 @@
"no-param-reassign": ["error", { "no-param-reassign": ["error", {
"props": true, "props": true,
"ignorePropertyModificationsFor": ["state", "acc"] "ignorePropertyModificationsFor": ["state", "acc"]
}] }],
}, },
"globals": { "globals": {
"CONFIG": true "CONFIG": true

6
package-lock.json generated
View File

@ -10307,9 +10307,9 @@
} }
}, },
"source-map-support": { "source-map-support": {
"version": "0.5.10", "version": "0.5.16",
"resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.10.tgz", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz",
"integrity": "sha512-YfQ3tQFTK/yzlGJuX8pTwa4tifQj4QS2Mj7UegOu8jAz59MqIiMGPXxQhVQiIMNzayuUSF/jEuVnfFF5JqybmQ==", "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==",
"requires": { "requires": {
"buffer-from": "^1.0.0", "buffer-from": "^1.0.0",
"source-map": "^0.6.0" "source-map": "^0.6.0"

View File

@ -5,12 +5,13 @@
"main": "src/app.js", "main": "src/app.js",
"scripts": { "scripts": {
"postinstall": "npm run migrate && npm run seed", "postinstall": "npm run migrate && npm run seed",
"start": "node dist/init.js", "start": "node -r source-map-support/register dist/init.js",
"webpack": "webpack --env=production --mode=production", "webpack": "webpack --env=production --mode=production",
"webpack-dev": "webpack --env=development --mode=development", "webpack-dev": "webpack --env=development --mode=development",
"webpack-watch": "webpack --progress --colors --watch --env=development --mode=development", "webpack-watch": "webpack --progress --colors --watch --env=development --mode=development",
"babel": "babel src -d dist", "babel": "babel src --source-maps -d dist",
"babel-watch": "babel src -w -d dist", "babel-watch": "babel src -w --source-maps -d dist",
"build": "babel src --source-maps -d dist && webpack --env=production --mode=production",
"eslint": "eslint src/", "eslint": "eslint src/",
"eslint-watch": "esw --watch src/", "eslint-watch": "esw --watch src/",
"knex": "knex", "knex": "knex",
@ -91,6 +92,7 @@
"react-dom": "^16.12.0", "react-dom": "^16.12.0",
"sharp": "^0.23.4", "sharp": "^0.23.4",
"showdown": "^1.9.1", "showdown": "^1.9.1",
"source-map-support": "^0.5.16",
"tough-cookie": "^3.0.1", "tough-cookie": "^3.0.1",
"tty-table": "^2.8.3", "tty-table": "^2.8.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",

View File

@ -9,7 +9,7 @@
"no-unused-vars": ["error", {"argsIgnorePattern": "^_"}], "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
"no-console": 0, "no-console": 0,
"indent": "off", "indent": "off",
"template-curly-spacing": "off",
"max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}], "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
"template-curly-spacing": "off"
} }
} }

View File

@ -271,8 +271,6 @@ async function updateActor(actor, scraped = false, scrapeSuccess = false) {
} }
async function mergeProfiles(profiles, actor) { async function mergeProfiles(profiles, actor) {
console.log(profiles);
const mergedProfile = profiles.reduce((prevProfile, profile) => { const mergedProfile = profiles.reduce((prevProfile, profile) => {
if (profile === null) { if (profile === null) {
return prevProfile; return prevProfile;

View File

@ -189,7 +189,7 @@ async function storePhotos(release, releaseId) {
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), { const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, `(${release.site.name}, ${releaseId}) "${release.title}"`), {
concurrency: 10, concurrency: 10,
}); }).filter(photo => photo);
const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`); const uniquePhotos = await filterHashDuplicates(metaFiles, 'releases', 'photo', `(${release.site.name}, ${releaseId}) "${release.title}"`);
const savedPhotos = await savePhotos(uniquePhotos, release, releaseId); const savedPhotos = await savePhotos(uniquePhotos, release, releaseId);
@ -244,7 +244,7 @@ async function storeAvatars(profile, actor) {
const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), { const metaFiles = await Promise.map(newPhotos, async (photoUrl, index) => fetchPhoto(photoUrl, index, actor.name), {
concurrency: 10, concurrency: 10,
}); }).filter(photo => photo);
const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name); const uniquePhotos = await filterHashDuplicates(metaFiles, 'actors', ['avatar', 'photo'], actor.name);
const [savedPhotos, avatarEntry] = await Promise.all([ const [savedPhotos, avatarEntry] = await Promise.all([

View File

@ -149,7 +149,7 @@ async function scrapeProfile(html, _url, actorName) {
if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]); if (bio['Shoe size']) profile.shoes = Number(bio['Shoe size'].split('|')[1]);
if (descriptionEl) profile.description = descriptionEl.textContent.trim(); if (descriptionEl) profile.description = descriptionEl.textContent.trim();
if (avatarEl) profile.avatar = `https:${avatarEl.dataset.src}`; if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
return profile; return profile;
} }

2
traxxx
View File

@ -1,2 +1,2 @@
#!/usr/bin/bash #!/usr/bin/bash
node ./dist/init.js "$@"; node -r source-map-support/register ./dist/init.js "$@";