test case fix and additon of numeric_double test

Author: MatloaItumeleng

src/test/scala/za/co/absa/standardization/TypeParserSuite.scala

@@ -19,16 +19,15 @@ package za.co.absa.standardization
 import org.apache.spark.sql.types._
 import org.scalatest.funsuite.AnyFunSuite
 import za.co.absa.spark.commons.test.SparkTestBase
-import za.co.absa.standardization.config.{BasicMetadataColumnsConfig, BasicStandardizationConfig, ErrorCodesConfig}
+import za.co.absa.standardization.config.{BasicMetadataColumnsConfig, BasicStandardizationConfig}
 import za.co.absa.standardization.schema.MetadataKeys
 import za.co.absa.standardization.types.{CommonTypeDefaults, TypeDefaults}
 import za.co.absa.standardization.udf.UDFLibrary
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.{DataFrame,SparkSession}
 import java.sql.{Date, Timestamp}
-import java.util.TimeZone
 
 class TypeParserSuite extends AnyFunSuite with SparkTestBase {
-
   import spark.implicits._
 
   private implicit val defaults: TypeDefaults = CommonTypeDefaults
@@ -37,38 +36,45 @@ class TypeParserSuite extends AnyFunSuite with SparkTestBase {
 
 
   private val testData: DataFrame = spark.createDataFrame(Seq(
-    (1,"2025-08-05","2025-08-05 12:34:56","250805"),
-    (2,"-INF","-INF","-INF"),
-    (3,"INF","INF","INF")
-  )).toDF("id","date","timestamp","custom_date")
+    (1,42.0,"2025-08-05","2025-08-05 12:34:56","250805"),
+    (2,-42.0,"-INF","-INF","-INF"),
+    (3,42.0,"INF","INF","INF")
+  )).toDF("id","numeric_double","date","timestamp","custom_date")
 
   test("Test TypeParser infinity handling for date and timestamp"){
     val schema: StructType = StructType(Seq(
       StructField("id",IntegerType, nullable = false),
+      StructField("numeric_double", DoubleType, nullable = true, Metadata.fromJson("""{"allow_infinity":true,"minus_infinity_symbol":"-INF","minus_infinity_value":"-1.7976931348623157E308","plus_infinity_symbol":"INF","plus_infinity_value":"1.7976931348623157E308"}""")),
       StructField("date",DateType, nullable = true, Metadata.fromJson("""{"pattern":"yyyy-MM-dd","minus_infinity_symbol":"-INF","minus_infinity_value":"1000-01-01","plus_infinity_symbol":"INF","plus_infinity_value":"9999-12-31"}""")),
       StructField("timestamp",TimestampType, nullable = true, Metadata.fromJson("""{"pattern":"yyyy-MM-dd HH:mm:ss","minus_infinity_symbol":"-INF","minus_infinity_value":"1000-01-01 00:00:00","plus_infinity_symbol":"INF","plus_infinity_value":"9999-12-31 23:59:59"}""")),
       StructField("custom_date",DateType, nullable = true, Metadata.fromJson("""{"pattern":"yyMMdd","minus_infinity_symbol":"-INF","minus_infinity_value":"1000-01-01","plus_infinity_symbol":"INF","plus_infinity_value":"9999-12-31"}""")),
     ))
 
     val stdDF = Standardization.standardize(testData,schema,stdConfig).cache()
 
-    val results = stdDF.select("id","date", "timestamp", "custom_date","errCol").collect()
-
-
-    assert(results(0).getAs[Date](1) == Date.valueOf("2025-08-05"))
-    assert(results(0).getAs[Timestamp](2) == Timestamp.valueOf("2025-08-05 12:34:56"))
-    assert(results(0).getAs[Date](3) == Date.valueOf("2025-08-05"))
-    assert(results(0).getAs[Seq[ErrorMessage]]("errCol").isEmpty)
-
-    assert(results(1).getAs[Date](1) == Date.valueOf("1000-01-01"))
-    assert(results(1).getAs[Timestamp](2) == Timestamp.valueOf("1000-01-01 00:00:00"))
-    assert(results(1).getAs[Date](3) == Date.valueOf("1000-01-01"))
-    assert(results(1).getAs[Seq[ErrorMessage]]("errCol").isEmpty)
-
-    assert(results(2).getAs[Date](1) == Date.valueOf("9999-12-31"))
-    assert(results(2).getAs[Timestamp](2) == Timestamp.valueOf("9999-12-31 23:59:59"))
-    assert(results(2).getAs[Date](3) == Date.valueOf("9999-12-31"))
-    assert(results(2).getAs[Seq[ErrorMessage]]("errCol").isEmpty)
+    val results = stdDF.select("id","numeric_double","date", "timestamp", "custom_date","errCol").collect()
+
+
+    assert(results(0).getInt(0) == 1)
+    assert(results(0).getDouble(1) == 42.0)
+    assert(results(0).getDate(2) == Date.valueOf("2025-08-05"))
+    assert(results(0).getTimestamp(3) == Timestamp.valueOf("2025-08-05 12:34:56"))
+    assert(results(0).getDate(4) == Date.valueOf("2025-08-05"))
+    assert(results(0).getAs[Seq[ErrorMessage]](5).isEmpty)
+
+    assert(results(1).getInt(0) == 2)
+    assert(results(1).getDouble(1) == -1.7976931348623157E308)
+    assert(results(1).getDate(2) == Date.valueOf("1000-01-01"))
+    assert(results(1).getTimestamp(3) == Timestamp.valueOf("1000-01-01 00:00:00"))
+    assert(results(1).getDate(4) == Date.valueOf("1000-01-01"))
+    assert(results(1).getAs[Seq[ErrorMessage]](5).isEmpty)
+
+    assert(results(2).getInt(0) == 3)
+    assert(results(2).getDouble(1) == 1.7976931348623157E308)
+    assert(results(2).getDate(2) == Date.valueOf("9999-12-31"))
+    assert(results(2).getTimestamp(3) == Timestamp.valueOf("9999-12-31 23:59:59"))
+    assert(results(2).getDate(4) == Date.valueOf("9999-12-31"))
+    assert(results(1).getAs[Seq[ErrorMessage]](5).isEmpty)
 
   }
 }