Merge pull request #10 from colindean/colindean/drop-env-dump_gh4

Drops env dump from JSON output, introduces System.env wrapper

Author: colindean

src/main/scala/com/target/data_validator/ConfigVar.scala

@@ -1,6 +1,7 @@
 package com.target.data_validator
 
 import cats.syntax.functor._
+import com.target.data_validator.EnvironmentVariables.{Error, Inaccessible, Present, Unset}
 import com.typesafe.scalalogging.LazyLogging
 import io.circe.{Decoder, Json}
 import io.circe.generic.auto._
@@ -21,18 +22,23 @@ case class NameValue(name: String, value: Json) extends ConfigVar {
 }
 
 case class NameEnv(name: String, env: String) extends ConfigVar {
+
   override def addEntry(spark: SparkSession, varSub: VarSubstitution): Boolean = {
     val newEnv = getVarSub(env, name, varSub)
-    Option(System.getenv(newEnv)) match {
-      case None =>
+    EnvironmentVariables.get(newEnv) match {
+      case Inaccessible(message) => logger.error(message); true
+      case Error(message) => logger.error(message); true
+      case Unset => {
         val msg = s"Variable '$name' cannot be processed env variable '$newEnv' not found!"
         logger.error(msg)
         addEvent(ValidatorError(msg))
         true
-      case Some(v) =>
-        val resolvedEnvVar = getVarSubJson(JsonUtils.string2Json(v), name, varSub)
-        logger.debug(s"name: $name env: $env getEnv: $v resolvedEnvVar: $resolvedEnvVar")
+      }
+      case Present(value) => {
+        val resolvedEnvVar = getVarSubJson(JsonUtils.string2Json(value), name, varSub)
+        logger.info(s"name: $name env: $env getEnv: $value resolvedEnvVar: $resolvedEnvVar")
         varSub.add(name, resolvedEnvVar)
+      }
     }
   }
 }

src/main/scala/com/target/data_validator/EnvironmentVariables.scala

@@ -0,0 +1,58 @@
+package com.target.data_validator
+
+import scala.collection.mutable
+import scala.util.Try
+
+object EnvironmentVariables {
+  type MaybeEnvVar = Try[Option[String]]
+
+  val accessedEnvVars: mutable.Map[String, MaybeEnvVar] = mutable.Map.empty
+
+  def get(key: String): EnvVarResult = {
+    getWithHandlers(key)(
+      whenError = { case throwable: Throwable => Inaccessible(throwable) },
+      whenUnset = { Unset },
+      whenPresent = { Present }
+    )
+      .recover { case throwable: Throwable => Error(throwable) }
+      .get
+  }
+
+  def getWithHandlers[T](key: String)(
+    whenError: PartialFunction[Throwable, T],
+    whenUnset: => T,
+    whenPresent: String => T
+  ): Try[T] = {
+    tryGet(key)
+      .map(_.map(whenPresent).getOrElse(whenUnset))
+      .recover(whenError)
+  }
+
+  def tryGet(key: String): MaybeEnvVar = {
+    val result = Try(System.getenv(key)).map(Option(_))
+    accessedEnvVars += key -> result
+    result
+  }
+
+  sealed trait EnvVarResult {
+    def toString: String
+  }
+  case class Present(value: String) extends EnvVarResult {
+    override def toString: String = value
+  }
+  case class Inaccessible(message: String) extends EnvVarResult {
+    override val toString: String = s"<inaccessible: $message>"
+  }
+  object Inaccessible {
+    def apply(throwable: Throwable): Inaccessible = Inaccessible(throwable.getMessage)
+  }
+  case object Unset extends EnvVarResult {
+    override val toString: String = "<unset>"
+  }
+  case class Error(message: String) extends EnvVarResult {
+    override val toString: String = s"<error: $message>"
+  }
+  object Error {
+    def apply(throwable: Throwable): Error = Error(throwable.getMessage)
+  }
+}

src/main/scala/com/target/data_validator/Main.scala

@@ -3,7 +3,7 @@ package com.target.data_validator
 import java.util.Properties
 
 import com.typesafe.scalalogging.LazyLogging
-import org.apache.log4j.PropertyConfigurator
+import org.apache.log4j.{Level, Logger, PropertyConfigurator}
 import org.apache.spark.sql.SparkSession
 import scopt.OptionParser
 
@@ -93,6 +93,7 @@ object Main extends LazyLogging with EventLog {
     if (mainConfig.verbose) {
       logger.info("Verbose Flag detected")
       logger.info(s"Original config: $origConfig")
+      Logger.getRootLogger.setLevel(Level.DEBUG)
     }
 
     // Resolve config

src/main/scala/com/target/data_validator/ValidatorConfig.scala

@@ -2,6 +2,7 @@ package com.target.data_validator
 
 import java.net.InetAddress
 
+import com.target.data_validator.EnvironmentVariables.MaybeEnvVar
 import com.typesafe.scalalogging.LazyLogging
 import io.circe.Json
 import io.circe.generic.auto._
@@ -101,8 +102,12 @@ object ValidatorConfig {
   }
 
   private def envToJson: Json = {
-    val env = System.getenv.asScala.toList.map(x => (x._1, Json.fromString(x._2)))
-    Json.obj(env: _*)
+    def extractFromAccessionList(pair: (String, MaybeEnvVar)) = {
+      pair._1 -> Json.fromString(pair._2.map(_.getOrElse("<unset>")).getOrElse("<inaccessible>"))
+    }
+
+    val env = EnvironmentVariables.accessedEnvVars.map(extractFromAccessionList)
+    Json.obj(env.toSeq: _*)
   }
 
   private def runtimeInfoJson(spark: SparkSession): Json = {

src/test/scala/com/target/data_validator/EnvironmentVariablesSpec.scala

@@ -0,0 +1,31 @@
+package com.target.data_validator
+
+import com.target.data_validator.EnvironmentVariables.{Inaccessible, Present, Unset}
+import org.scalatest.{Matchers, WordSpec}
+
+class EnvironmentVariablesSpec extends WordSpec with Matchers {
+
+  "EnvironmentVariables" should {
+    "get envvars" when {
+      "an envvar exists" in {
+        EnvironmentVariables.get("HOME") should be(Present(System.getenv("HOME")))
+      }
+      "an envvar doesn't exist" in {
+        EnvironmentVariables.get("NOPE") should be(Unset)
+      }
+      "an envvar isn't an envvar" in {
+        EnvironmentVariables.get(null) shouldBe a[Inaccessible] // scalastyle:ignore
+      }
+    }
+    "log envvars" when {
+      "using get" in {
+        EnvironmentVariables.get("HOME")
+        EnvironmentVariables.accessedEnvVars.keySet should contain ("HOME")
+      }
+      "using tryGet" in {
+        EnvironmentVariables.tryGet("HOME")
+        EnvironmentVariables.accessedEnvVars.keySet should contain ("HOME")
+      }
+    }
+  }
+}