Merge pull request #175 from AbsaOSS/fix/logger

Fix issue setting up JSON formatted logging

Author: mirgee

vcxagency-node/src/api/api-health.js

@@ -1,3 +1,21 @@
+/**
+ * Copyright 2020 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict'
+
 const { asyncHandler } = require('./middleware')
 
 module.exports = function (app) {

vcxagency-node/src/configuration/app-config-loader.js

@@ -18,58 +18,27 @@
 
 const path = require('path')
 const dotenv = require('dotenv')
+const fs = require('fs')
+const assert = require('assert')
 
 const appConfigName = process.env.APP_CONFIG
 if (appConfigName) {
   const pathToConfig = path.resolve(__dirname, `../../config/${appConfigName}.env`)
+  assert(fs.existsSync(pathToConfig), `File ${pathToConfig} not found.`)
   console.log(`App configuration will be loaded from ${pathToConfig}.`)
   dotenv.config({ path: pathToConfig })
 } else {
   console.log('App configuration will be loaded from supplied environment variables.')
 }
 
-function buildAppConfigFromEnvVariables () {
-  const appConfig = {
-    LOG_LEVEL: process.env.LOG_LEVEL,
-    LOG_ENABLE_INDYSDK: process.env.LOG_ENABLE_INDYSDK,
-    LOG_JSON_TO_CONSOLE: process.env.LOG_JSON_TO_CONSOLE,
-    LOG_HEALTH_REQUESTS: process.env.LOG_HEALTH_REQUESTS,
-    DEV_MODE: process.env.DEV_MODE,
-
-    SERVER_PORT: process.env.SERVER_PORT,
-    SERVER_HOSTNAME: process.env.SERVER_HOSTNAME,
-    SERVER_MAX_REQUEST_SIZE_KB: process.env.SERVER_MAX_REQUEST_SIZE_KB,
-    SERVER_ENABLE_TLS: process.env.SERVER_ENABLE_TLS,
-    CERTIFICATE_PATH: process.env.CERTIFICATE_PATH,
-    CERTIFICATE_KEY_PATH: process.env.CERTIFICATE_KEY_PATH,
-
-    AGENCY_WALLET_NAME: process.env.AGENCY_WALLET_NAME,
-    AGENCY_DID: process.env.AGENCY_DID,
-    AGENCY_SEED_SECRET: process.env.AGENCY_SEED_SECRET,
-    AGENCY_WALLET_KEY_SECRET: process.env.AGENCY_WALLET_KEY_SECRET,
-
-    REDIS_URL: process.env.REDIS_URL,
-    AGENCY_TYPE: process.env.AGENCY_TYPE,
-
-    MYSQL_HOST: process.env.MYSQL_HOST,
-    MYSQL_PORT: process.env.MYSQL_PORT,
-    MYSQL_ACCOUNT: process.env.MYSQL_ACCOUNT,
-    MYSQL_PASSWORD_SECRET: process.env.MYSQL_PASSWORD_SECRET,
-    MYSQL_DATABASE_APPLICATION: process.env.MYSQL_DATABASE_APPLICATION,
-    MYSQL_DATABASE_WALLET: process.env.MYSQL_DATABASE_WALLET,
-    MYSQL_DATABASE_WALLET_CONNECTION_LIMIT: process.env.MYSQL_DATABASE_WALLET_CONNECTION_LIMIT,
-
-    AWS_S3_PATH_CERT: process.env.AWS_S3_PATH_CERT,
-    AWS_S3_BUCKET_CERT: process.env.AWS_S3_BUCKET_CERT,
-    AWS_S3_PATH_CERT_KEY: process.env.AWS_S3_PATH_CERT_KEY,
-
-    ECS_CONTAINER_METADATA_URI_V4: process.env.ECS_CONTAINER_METADATA_URI_V4,
-    WEBHOOK_RESPONSE_TIMEOUT_MS: process.env.WEBHOOK_RESPONSE_TIMEOUT_MS,
-    EXPLAIN_QUERIES: process.env.EXPLAIN_QUERIES
+function loadEnvVariables (envVariables) {
+  const inputConfig = {}
+  for (const envName of envVariables) {
+    inputConfig[envName] = process.env[envName]
   }
-  appConfig.SERVER_MAX_REQUEST_SIZE_KB = parseInt(appConfig.SERVER_MAX_REQUEST_SIZE_KB)
-  appConfig.MYSQL_DATABASE_WALLET_CONNECTION_LIMIT = parseInt(appConfig.MYSQL_DATABASE_WALLET_CONNECTION_LIMIT)
-  return appConfig
+  return inputConfig
 }
 
-module.exports = { buildAppConfigFromEnvVariables }
+module.exports = {
+  loadEnvVariables
+}

vcxagency-node/src/configuration/app-config.js

@@ -17,6 +17,7 @@
 'use strict'
 
 const Joi = require('joi')
+const { loadEnvVariables } = require('./app-config-loader')
 
 function stringifyAndHideSensitive (appConfig) {
   function hideSecrets (key, value) {
@@ -37,72 +38,127 @@ function stringifyAndHideSensitive (appConfig) {
 
 const MB_AS_KB = 1024
 
-const configValidation = Joi.object().keys({
-  LOG_LEVEL: Joi.string().valid('silly', 'debug', 'info', 'warn', 'error'),
-  LOG_ENABLE_INDYSDK: Joi.boolean().default(false),
-  LOG_JSON_TO_CONSOLE: Joi.boolean().default(true),
-  LOG_HEALTH_REQUESTS: Joi.boolean().default(false),
-  DEV_MODE: Joi.boolean().default(false),
-
-  SERVER_HOSTNAME: Joi.string().default('0.0.0.0'),
-  SERVER_PORT: Joi.number().integer().min(1025).max(65535).required(),
-  SERVER_ENABLE_TLS: Joi.boolean().default(true),
-  SERVER_MAX_REQUEST_SIZE_KB: Joi.number().integer().min(1).max(MB_AS_KB * 10).default(512),
-  CERTIFICATE_PATH: Joi.string(),
-  CERTIFICATE_KEY_PATH: Joi.string(),
-
-  AGENCY_WALLET_NAME: Joi.string().required(),
-  AGENCY_WALLET_KEY_SECRET: Joi.string().min(20).required(),
-  AGENCY_SEED_SECRET: Joi.string().min(20).required(),
-  AGENCY_DID: Joi.string().required(),
-
-  REDIS_URL: Joi.string(),
-  AGENCY_TYPE: Joi.string().valid('enterprise', 'client').required(),
-
-  MYSQL_HOST: Joi.string().required(),
-  MYSQL_PORT: Joi.number().integer().min(1025).max(65535).default(3306).required(),
-  MYSQL_ACCOUNT: Joi.string().required(),
-  MYSQL_PASSWORD_SECRET: Joi.string().required(),
-  MYSQL_DATABASE_APPLICATION: Joi.string().required(),
-  MYSQL_DATABASE_WALLET: Joi.string().required(),
-  MYSQL_DATABASE_WALLET_CONNECTION_LIMIT: Joi.number().integer().min(1).max(100).default(50),
-
-  AWS_S3_PATH_CERT: Joi.string(),
-  AWS_S3_BUCKET_CERT: Joi.string(),
-  AWS_S3_PATH_CERT_KEY: Joi.string(),
-
-  ECS_CONTAINER_METADATA_URI_V4: Joi.string().uri(),
-  WEBHOOK_RESPONSE_TIMEOUT_MS: Joi.number().default(1000),
-  EXPLAIN_QUERIES: Joi.boolean().default(false)
-})
-
-function validateFinalConfig (appConfig) {
+function _getLoggingValidationRules () {
+  return {
+    LOG_LEVEL: Joi.string().valid('trace', 'debug', 'info', 'warn', 'error').default('info'),
+    LOG_JSON_TO_CONSOLE: Joi.boolean().default(true),
+    LOG_ENABLE_INDYSDK: Joi.boolean().default(false),
+    DISABLE_COLOR_LOGS: Joi.boolean().default(false),
+    LOG_HEALTH_REQUESTS: Joi.boolean().default(false),
+    EXPLAIN_QUERIES: Joi.boolean().default(false),
+    DANGEROUS_HTTP_DETAILS: Joi.boolean().default(false),
+    ECS_CONTAINER_METADATA_URI_V4: Joi.string().uri()
+  }
+}
+
+function _mysqlValidationRules () {
+  return {
+    MYSQL_HOST: Joi.string().required(),
+    MYSQL_PORT: Joi.number().integer().min(1025).max(65535).default(3306).required(),
+    MYSQL_ACCOUNT: Joi.string().required(),
+    MYSQL_PASSWORD_SECRET: Joi.string().required(),
+    MYSQL_DATABASE_APPLICATION: Joi.string().required(),
+    MYSQL_DATABASE_WALLET: Joi.string().required(),
+    MYSQL_DATABASE_WALLET_CONNECTION_LIMIT: Joi.number().integer().min(1).max(100).default(50)
+  }
+}
+
+function _getTlsValidationRules () {
+  return {
+    SERVER_ENABLE_TLS: Joi.boolean().default(true),
+    AWS_S3_BUCKET_CERT: Joi.string(),
+    CERTIFICATE_PATH: Joi.string(),
+    CERTIFICATE_KEY_PATH: Joi.string(),
+    AWS_S3_PATH_CERT: Joi.string(),
+    AWS_S3_PATH_CERT_KEY: Joi.string()
+  }
+}
+
+function _getServerValidationRules () {
+  return {
+    SERVER_PORT: Joi.number().integer().min(1025).max(65535).required(),
+    SERVER_HOSTNAME: Joi.string().default('0.0.0.0'),
+    SERVER_MAX_REQUEST_SIZE_KB: Joi.number().integer().min(1).max(MB_AS_KB * 10).default(512)
+  }
+}
+
+function _setupWalletValidationRules () {
+  return {
+    AGENCY_SEED_SECRET: Joi.string().min(20).required()
+  }
+}
+
+function _walletValidationRules () {
+  return {
+    AGENCY_WALLET_NAME: Joi.string().required(),
+    AGENCY_WALLET_KEY_SECRET: Joi.string().min(20).required()
+  }
+}
+
+function _applicationValidationRules () {
+  return {
+    REDIS_URL: Joi.string().uri(),
+    AGENCY_TYPE: Joi.string().valid('enterprise', 'client').required(),
+    WEBHOOK_RESPONSE_TIMEOUT_MS: Joi.number().default(1000)
+  }
+}
+
+function _extraValidationTls (appConfig) {
+  if (appConfig.SERVER_ENABLE_TLS) {
+    if (!appConfig.CERTIFICATE_PATH || !appConfig.CERTIFICATE_KEY_PATH) {
+      throw new Error('Valid certificate and key paths must be specified when TLS enabled!')
+    }
+  }
+}
+
+function _extraValidationByAgencyType (appConfig) {
   if (appConfig.AGENCY_TYPE === 'client') {
     if (!appConfig.REDIS_URL) {
       throw new Error('Configuration for agency of type \'client\' must have REDIS_URL specified.')
     }
   }
-  function validateTls () {
-    if (appConfig.SERVER_ENABLE_TLS === true) {
-      if (!appConfig.CERTIFICATE_PATH || !appConfig.CERTIFICATE_KEY_PATH) {
-        throw new Error('Valid certificate and key paths must be specified when TLS enabled!')
-      }
+}
+
+const OP_MODES = {
+  RUN_SERVER: {
+    name: 'run-server',
+    postValidations: [_extraValidationTls, _extraValidationByAgencyType],
+    joiValidationBody: {
+      ..._getLoggingValidationRules(),
+      ..._mysqlValidationRules(),
+      ..._getTlsValidationRules(),
+      ..._getServerValidationRules(),
+      ..._setupWalletValidationRules(),
+      ..._walletValidationRules(),
+      ..._applicationValidationRules(),
+      AGENCY_DID: Joi.string().required(),
+      DEV_MODE: Joi.boolean().default(false),
+      ECS_CONTAINER_METADATA_URI_V4: Joi.string().uri()
     }
   }
-  validateTls()
 }
 
 async function validateAppConfig (appConfig) {
-  const { value: effectiveConfig, error } = configValidation.validate(appConfig)
+  const operationModeInfo = OP_MODES.RUN_SERVER
+  const { postValidations } = operationModeInfo
+  const { value: effectiveConfig, error } = Joi.object().keys(operationModeInfo.joiValidationBody).validate(appConfig)
   if (error) {
     throw new Error(`Application configuration is not valid. Details ${stringifyAndHideSensitive(error)}`)
   }
-  validateFinalConfig(effectiveConfig)
+  for (const postValidation of postValidations) {
+    postValidation(effectiveConfig)
+  }
   return effectiveConfig
 }
 
+async function loadConfiguration () {
+  const envVariables = Object.keys(OP_MODES.RUN_SERVER.joiValidationBody)
+  return loadEnvVariables(envVariables)
+}
+
 module.exports = {
   validateAppConfig,
-  validateFinalConfig,
-  stringifyAndHideSensitive
+  stringifyAndHideSensitive,
+  loadConfiguration,
+  OP_MODES
 }

vcxagency-node/src/setup/app.js vcxagency-node/src/execution/app.jsvcxagency-node/src/setup/app.js renamed to vcxagency-node/src/execution/app.js

@@ -0,0 +1,62 @@
+/**
+ * Copyright 2020 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict'
+
+const express = require('express')
+const logger = require('./../logging/logger-builder')(__filename)
+const { fetchServerCertsFromS3 } = require('../scripts/download-certs')
+const { buildApplication } = require('./app')
+const { createWebServer, setupExpressApp } = require('./server')
+
+async function startServer (appConfig) {
+  try {
+    process.on('SIGTERM', signal => {
+      logger.warn(`Process ${process.pid} [startServer execution mode] received a SIGTERM signal.`)
+      process.exit(0)
+    })
+
+    process.on('SIGINT', signal => {
+      logger.warn(`Process ${process.pid} [startServer execution mode] has been interrupted.`)
+      process.exit(0)
+    })
+
+    if (appConfig.SERVER_ENABLE_TLS) {
+      logger.info('Fetching certificates/keys to serve.')
+      await fetchServerCertsFromS3(appConfig)
+    }
+
+    logger.debug('Going to build application internals.')
+    const application = await buildApplication(appConfig)
+
+    const expressApp = express()
+    expressApp.set('app-name', 'agency')
+
+    logger.debug('Going to build http/https server.')
+    const httpServer = createWebServer(expressApp, appConfig.SERVER_ENABLE_TLS, appConfig.CERTIFICATE_PATH, appConfig.CERTIFICATE_KEY_PATH, logger)
+    await setupExpressApp(expressApp, application, appConfig)
+
+    logger.debug(`Going to listen on port ${appConfig.SERVER_PORT}`)
+    httpServer.listen(appConfig.SERVER_PORT, () => logger.info(`------------ Listening on port ${appConfig.SERVER_PORT} ------------`))
+  } catch (err) {
+    logger.error(`Unhandled error. Application will be terminated. ${err.stack}`)
+    process.exit(255)
+  }
+}
+
+module.exports = {
+  startServer
+}

vcxagency-node/src/execution/run.js

@@ -47,16 +47,6 @@ function createWebServer (expressApp, enableTls, tlsCertPath, tlsKeyPath, logger
   }
 }
 
-function reloadTrustedCerts (logger) {
-  const TRUSTED_CA_CERTS_PATH = '/etc/ssl/certs/ca-certificates.crt'
-  if (fs.existsSync(TRUSTED_CA_CERTS_PATH)) {
-    https.globalAgent.options.ca = fs.readFileSync(TRUSTED_CA_CERTS_PATH)
-    logger.warn(`Loaded additional trusted CA certificates from ${TRUSTED_CA_CERTS_PATH}`)
-  } else {
-    logger.warn('No additional trusted CA certificates were loaded.')
-  }
-}
-
 async function setupExpressApp (expressApp, application, appConfig) {
   const { entityForwardAgent, serviceNewMessages } = application
   logger.info('Setting up express endpoints and middleware.')
@@ -106,6 +96,5 @@ async function setupExpressApp (expressApp, application, appConfig) {
 
 module.exports = {
   setupExpressApp,
-  createWebServer,
-  reloadTrustedCerts
+  createWebServer
 }