[SPARK-51871] Improve SQLTests to check column names

dongjoon-hyun · dongjoon-hyun · commit 3137801f8f62 · 2025-04-30T21:26:00.000-07:00
diff --git a/Tests/SparkConnectTests/Resources/queries/binary.sql.answer b/Tests/SparkConnectTests/Resources/queries/binary.sql.answer
@@ -1 +1,5 @@
-[[61 62 63]]
++----------+
+|abc       |
++----------+
+|[61 62 63]|
++----------+
diff --git a/Tests/SparkConnectTests/Resources/queries/cache.sql.answer b/Tests/SparkConnectTests/Resources/queries/cache.sql.answer
@@ -1 +1,4 @@
-
+++
+||
+++
+++
diff --git a/Tests/SparkConnectTests/Resources/queries/clear_cache.sql.answer b/Tests/SparkConnectTests/Resources/queries/clear_cache.sql.answer
@@ -1 +1,4 @@
-
+++
+||
+++
+++
diff --git a/Tests/SparkConnectTests/Resources/queries/date.sql.answer b/Tests/SparkConnectTests/Resources/queries/date.sql.answer
@@ -1 +1,5 @@
-[2025-03-15 00:00:00 +0000]
++-----------------+
+|DATE '2025-03-15'|
++-----------------+
+|2025-03-15       |
++-----------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_database.sql.answer b/Tests/SparkConnectTests/Resources/queries/describe_database.sql.answer
@@ -1,5 +1,9 @@
-[Catalog Name,spark_catalog]
-[Namespace Name,default]
-[Comment,default database]
-[Location,*]
-[Owner,*]
++--------------+----------------------------------------+
+|info_name     |info_value                              |
++--------------+----------------------------------------+
+|Catalog Name  |spark_catalog                           |
+|Namespace Name|default                                 |
+|Comment       |default database                        |
+|Location      |*|
+|Owner         |*                                     |
++--------------+----------------------------------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_function.sql.answer b/Tests/SparkConnectTests/Resources/queries/describe_function.sql.answer
@@ -1,3 +1,7 @@
-[Function: abs]
-[Class: org.apache.spark.sql.catalyst.expressions.Abs]
-[Usage: abs(expr) - Returns the absolute value of the numeric or interval value.]
++-------------------------------------------------------------------------------+
+|function_desc                                                                  |
++-------------------------------------------------------------------------------+
+|Function: abs                                                                  |
+|Class: org.apache.spark.sql.catalyst.expressions.Abs                           |
+|Usage: abs(expr) - Returns the absolute value of the numeric or interval value.|
++-------------------------------------------------------------------------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_query.sql.answer b/Tests/SparkConnectTests/Resources/queries/describe_query.sql.answer
@@ -1,3 +1,7 @@
-[id,int,null]
-[name,string,null]
-[salary,double,null]
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|id      |int      |NULL   |
+|name    |string   |NULL   |
+|salary  |double   |NULL   |
++--------+---------+-------+
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_table.sql.answer b/Tests/SparkConnectTests/Resources/queries/describe_table.sql.answer
@@ -1 +1,5 @@
-[col,int,null]
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|col     |int      |NULL   |
++--------+---------+-------+
diff --git a/Tests/SparkConnectTests/Resources/queries/explain.sql.answer b/Tests/SparkConnectTests/Resources/queries/explain.sql.answer
@@ -1,22 +1,5 @@
-[== Parsed Logical Plan ==
-'Aggregate ['k], ['k, unresolvedalias('sum('v))]
-+- SubqueryAlias t
-   +- LocalRelation [k#, v#]
-
-== Analyzed Logical Plan ==
-k: int, sum(v): bigint
-Aggregate [k#], [k#, sum(v#) AS sum(v)#]
-+- SubqueryAlias t
-   +- LocalRelation [k#, v#]
-
-== Optimized Logical Plan ==
-Aggregate [k#], [k#, sum(v#) AS sum(v)#]
-+- LocalRelation [k#, v#]
-
-== Physical Plan ==
-AdaptiveSparkPlan isFinalPlan=false
-+- HashAggregate(keys=[k#], functions=[sum(v#)], output=[k#, sum(v)#])
-   +- Exchange hashpartitioning(k#, 200), ENSURE_REQUIREMENTS, [plan_id=]
-      +- HashAggregate(keys=[k#], functions=[partial_sum(v#)], output=[k#, sum#])
-         +- LocalTableScan [k#, v#]
-]
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|plan                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|== Parsed Logical Plan ==\n'Aggregate ['k], ['k, unresolvedalias('sum('v))]\n+- SubqueryAlias t\n   +- LocalRelation [k#, v#]\n\n== Analyzed Logical Plan ==\nk: int, sum(v): bigint\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- SubqueryAlias t\n   +- LocalRelation [k#, v#]\n\n== Optimized Logical Plan ==\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- LocalRelation [k#, v#]\n\n== Physical Plan ==\nAdaptiveSparkPlan isFinalPlan=false\n+- HashAggregate(keys=[k#], functions=[sum(v#)], output=[k#, sum(v)#])\n   +- Exchange hashpartitioning(k#, 200), ENSURE_REQUIREMENTS, [plan_id=]\n      +- HashAggregate(keys=[k#], functions=[partial_sum(v#)], output=[k#, sum#])\n         +- LocalTableScan [k#, v#]\n|
++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/floating.sql.answer b/Tests/SparkConnectTests/Resources/queries/floating.sql.answer
@@ -1 +1,5 @@
-[1.0,-2.0,3.0,-4.0,inf,nan,inf,nan]
++---+----+---+----+--------+---+--------+---+
+|1.0|-2.0|3.0|-4.0|inf     |NaN|inf     |NaN|
++---+----+---+----+--------+---+--------+---+
+|1.0|-2.0|3.0|-4.0|Infinity|NaN|Infinity|NaN|
++---+----+---+----+--------+---+--------+---+
diff --git a/Tests/SparkConnectTests/Resources/queries/integral.sql.answer b/Tests/SparkConnectTests/Resources/queries/integral.sql.answer
@@ -1 +1,5 @@
-[127,-128,32767,-32768,2147483647,-2147483648,9223372036854775807,-9223372036854775808]
++---+----+-----+------+----------+-----------+-------------------+--------------------+
+|127|-128|32767|-32768|2147483647|-2147483648|9223372036854775807|-9223372036854775808|
++---+----+-----+------+----------+-----------+-------------------+--------------------+
+|127|-128|32767|-32768|2147483647|-2147483648|9223372036854775807|-9223372036854775808|
++---+----+-----+------+----------+-----------+-------------------+--------------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/pipesyntax.sql.answer b/Tests/SparkConnectTests/Resources/queries/pipesyntax.sql.answer
@@ -1,2 +1,6 @@
-[0,0]
-[1,2]
++---+------+
+|col|result|
++---+------+
+|0  |0     |
+|1  |2     |
++---+------+
diff --git a/Tests/SparkConnectTests/Resources/queries/select.sql.answer b/Tests/SparkConnectTests/Resources/queries/select.sql.answer
@@ -1 +1,5 @@
-[1]
++---+
+|1  |
++---+
+|1  |
++---+
diff --git a/Tests/SparkConnectTests/Resources/queries/show_databases.sql.answer b/Tests/SparkConnectTests/Resources/queries/show_databases.sql.answer
@@ -1 +1,5 @@
-[default]
++---------+
+|namespace|
++---------+
+|default  |
++---------+
diff --git a/Tests/SparkConnectTests/Resources/queries/show_tables.sql.answer b/Tests/SparkConnectTests/Resources/queries/show_tables.sql.answer
@@ -1 +1,5 @@
-[,testcache,true]
++---------+---------+-----------+
+|namespace|tableName|isTemporary|
++---------+---------+-----------+
+|         |testcache|true       |
++---------+---------+-----------+
diff --git a/Tests/SparkConnectTests/Resources/queries/string.sql.answer b/Tests/SparkConnectTests/Resources/queries/string.sql.answer
@@ -1 +1,5 @@
-[abc,def]
++---+---+
+|abc|def|
++---+---+
+|abc|def|
++---+---+
diff --git a/Tests/SparkConnectTests/Resources/queries/struct.sql.answer b/Tests/SparkConnectTests/Resources/queries/struct.sql.answer
@@ -1 +1,5 @@
-[{1},{2,{3}}]
++---------+--------------------+
+|struct(1)|struct(2, struct(3))|
++---------+--------------------+
+|{1}      |{2, {3}}            |
++---------+--------------------+
diff --git a/Tests/SparkConnectTests/Resources/queries/uncache.sql.answer b/Tests/SparkConnectTests/Resources/queries/uncache.sql.answer
@@ -1 +1,4 @@
-
+++
+||
+++
+++
diff --git a/Tests/SparkConnectTests/SQLTests.swift b/Tests/SparkConnectTests/SQLTests.swift
@@ -20,7 +20,7 @@
 import Foundation
 import Testing
 
-import SparkConnect
+@testable import SparkConnect
 
 /// A test suite for various SQL statements.
 struct SQLTests {
@@ -89,10 +89,11 @@ struct SQLTests {
       }
 
       let sql = try String(contentsOf: URL(fileURLWithPath: "\(path)/\(name)"), encoding: .utf8)
-      let answer = cleanUp(try await spark.sql(sql).collect().map { $0.toString() }.joined(separator: "\n"))
+      let result = try await spark.sql(sql).showString(Int32.max, 0, false).collect()[0].get(0) as! String
+      let answer = cleanUp(result.trimmingCharacters(in: .whitespacesAndNewlines))
       if (regenerateGoldenFiles) {
         let path = "\(FileManager.default.currentDirectoryPath)/Tests/SparkConnectTests/Resources/queries/\(name).answer"
-        fm.createFile(atPath: path, contents: (answer + "\n").data(using: .utf8)!, attributes: nil)
+        fm.createFile(atPath: path, contents: answer.data(using: .utf8)!, attributes: nil)
       } else {
         let expected = cleanUp(try String(contentsOf: URL(fileURLWithPath: "\(path)/\(name).answer"), encoding: .utf8))
         #expect(answer == expected.trimmingCharacters(in: .whitespacesAndNewlines))

-Original file line number
+Diff line change
@@ @@ -1 +1,5 @@ @@
 -[[61 62 63]]
 ++----------+
 +|abc       |
 ++----------+
 +|[61 62 63]|
 ++----------+