diff --git a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala index 1912e982b9..317fa1d22e 100644 --- a/spark/src/test/scala/org/apache/comet/CometCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCastSuite.scala @@ -33,7 +33,6 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, DataType, DataTypes, DecimalType, IntegerType, LongType, ShortType, StringType, StructField, StructType} -import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.Compatible @@ -575,8 +574,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { // CAST from StringType test("cast StringType to BooleanType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) val testValues = (Seq("TRUE", "True", "true", "FALSE", "False", "false", "1", "0", "", null) ++ gen.generateStrings(dataSize, "truefalseTRUEFALSEyesno10" + whitespaceChars, 8)).toDF("a") @@ -617,8 +614,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { ) test("cast StringType to ByteType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) // test with hand-picked values castTest(castStringToIntegralInputs.toDF("a"), DataTypes.ByteType) // fuzz test @@ -626,8 +621,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("cast StringType to ShortType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) // test with hand-picked values castTest(castStringToIntegralInputs.toDF("a"), DataTypes.ShortType) // fuzz test @@ -635,8 +628,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("cast StringType to IntegerType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) // test with hand-picked values castTest(castStringToIntegralInputs.toDF("a"), DataTypes.IntegerType) // fuzz test @@ -644,8 +635,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("cast StringType to LongType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) // test with hand-picked values castTest(castStringToIntegralInputs.toDF("a"), DataTypes.LongType) // fuzz test @@ -707,8 +696,6 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("cast StringType to DateType") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) val validDates = Seq( "262142-01-01", "262142-01-01 ", @@ -1295,10 +1282,13 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } else { if (CometSparkSessionExtensions.isSpark40Plus) { // for Spark 4 we expect to sparkException carries the message + // spark4 also adds a new SQLSTATE message assert( sparkException.getMessage .replace(".WITH_SUGGESTION] ", "]") - .startsWith(cometMessage)) + .startsWith(cometMessage.replace( + """If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error.""", + ""))) } else { // for Spark 3.4 we expect to reproduce the error message exactly assert(cometMessage == sparkMessage)