improve tests

Author: andygrove

docs/source/user-guide/latest/compatibility.md

@@ -105,15 +105,14 @@ Cast operations in Comet fall into three levels of support:
 <!-- prettier-ignore-end -->
 
 **Notes:**
-
 - **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
 - **double -> decimal**: There can be rounding differences
 - **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **float -> decimal**: There can be rounding differences
 - **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **string -> date**: Only supports years between 262143 BC and 262142 AD
 - **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
-  or strings containing null bytes (e.g \\u0000)
+or strings containing null bytes (e.g \\u0000)
 - **string -> timestamp**: Not all valid formats are supported
 <!--END:CAST_LEGACY_TABLE-->
 
@@ -140,15 +139,14 @@ Cast operations in Comet fall into three levels of support:
 <!-- prettier-ignore-end -->
 
 **Notes:**
-
 - **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
 - **double -> decimal**: There can be rounding differences
 - **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **float -> decimal**: There can be rounding differences
 - **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **string -> date**: Only supports years between 262143 BC and 262142 AD
 - **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
-  or strings containing null bytes (e.g \\u0000)
+or strings containing null bytes (e.g \\u0000)
 - **string -> timestamp**: Not all valid formats are supported
 <!--END:CAST_TRY_TABLE-->
 
@@ -175,15 +173,14 @@ Cast operations in Comet fall into three levels of support:
 <!-- prettier-ignore-end -->
 
 **Notes:**
-
 - **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
 - **double -> decimal**: There can be rounding differences
 - **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **float -> decimal**: There can be rounding differences
 - **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
 - **string -> date**: Only supports years between 262143 BC and 262142 AD
 - **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
-  or strings containing null bytes (e.g \\u0000)
+or strings containing null bytes (e.g \\u0000)
 - **string -> timestamp**: ANSI mode not supported
 <!--END:CAST_ANSI_TABLE-->
 

spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala

@@ -193,10 +193,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
         withTempDir { dir =>
           val path = new Path(dir.toURI.toString, "test.parquet")
           makeParquetFileAllPrimitiveTypes(path, dictionaryEnabled = dictionaryEnabled, 10000)
-          withSQLConf(CometConf.COMET_SCAN_ALLOW_INCOMPATIBLE.key -> "true") {
-            withParquetTable(path.toString, "tbl") {
-              checkSparkAnswerAndOperator("select * FROM tbl WHERE _2 > 100")
-            }
+          withParquetTable(path.toString, "tbl") {
+            checkSparkAnswerAndOperator("select * FROM tbl WHERE _2 > 100")
           }
         }
       }
@@ -207,28 +205,16 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     // this test requires native_comet scan due to unsigned u8/u16 issue
     withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) {
       Seq(true, false).foreach { dictionaryEnabled =>
-        withSQLConf(CometConf.COMET_SCAN_ALLOW_INCOMPATIBLE.key -> "true") {
-          withTempDir { dir =>
-            val path = new Path(dir.toURI.toString, "testuint.parquet")
-            makeParquetFileAllPrimitiveTypes(
-              path,
-              dictionaryEnabled = dictionaryEnabled,
-              Byte.MinValue,
-              Byte.MaxValue)
-            withParquetTable(path.toString, "tbl") {
-              val qry = "select _9 from tbl order by _11"
-              if (usingDataSourceExec(conf)) {
-                // need to convert the values to unsigned values
-                val expected = (Byte.MinValue to Byte.MaxValue)
-                  .map(v => {
-                    if (v < 0) Byte.MaxValue.toShort - v else v
-                  })
-                  .toDF("a")
-                checkAnswer(sql(qry), expected)
-              } else {
-                checkSparkAnswerAndOperator(qry)
-              }
-            }
+        withTempDir { dir =>
+          val path = new Path(dir.toURI.toString, "testuint.parquet")
+          makeParquetFileAllPrimitiveTypes(
+            path,
+            dictionaryEnabled = dictionaryEnabled,
+            Byte.MinValue,
+            Byte.MaxValue)
+          withParquetTable(path.toString, "tbl") {
+            val qry = "select _9 from tbl order by _11"
+            checkSparkAnswerAndOperator(qry)
           }
         }
       }