[SPARK-51743] Add describe_(database|table), show_(database|table), explain sql test and answer files

dongjoon-hyun · dongjoon-hyun · commit 3fad56f97d50 · 2025-04-08T14:07:42.000+09:00
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_database.sql b/Tests/SparkConnectTests/Resources/queries/describe_database.sql
@@ -0,0 +1 @@
+DESCRIBE DATABASE default
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_database.sql.json b/Tests/SparkConnectTests/Resources/queries/describe_database.sql.json
@@ -0,0 +1 @@
+[["Catalog Name","spark_catalog"],["Namespace Name","default"],["Comment","default database"],["Location","file:\/opt\/spark\/work-dir\/spark-warehouse"],["Owner","185"]]
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_table.sql b/Tests/SparkConnectTests/Resources/queries/describe_table.sql
@@ -0,0 +1 @@
+DESCRIBE TABLE testcache
diff --git a/Tests/SparkConnectTests/Resources/queries/describe_table.sql.json b/Tests/SparkConnectTests/Resources/queries/describe_table.sql.json
@@ -0,0 +1 @@
+[["col","int",null]]
diff --git a/Tests/SparkConnectTests/Resources/queries/explain.sql b/Tests/SparkConnectTests/Resources/queries/explain.sql
@@ -0,0 +1 @@
+EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k
diff --git a/Tests/SparkConnectTests/Resources/queries/explain.sql.json b/Tests/SparkConnectTests/Resources/queries/explain.sql.json
@@ -0,0 +1 @@
+[["== Parsed Logical Plan ==\n'Aggregate ['k], ['k, unresolvedalias('sum('v))]\n+- SubqueryAlias t\n   +- LocalRelation [k#, v#]\n\n== Analyzed Logical Plan ==\nk: int, sum(v): bigint\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- SubqueryAlias t\n   +- LocalRelation [k#, v#]\n\n== Optimized Logical Plan ==\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- LocalRelation [k#, v#]\n\n== Physical Plan ==\nAdaptiveSparkPlan isFinalPlan=false\n+- HashAggregate(keys=[k#], functions=[sum(v#)], output=[k#, sum(v)#])\n   +- Exchange hashpartitioning(k#, 200), ENSURE_REQUIREMENTS, [plan_id=]\n      +- HashAggregate(keys=[k#], functions=[partial_sum(v#)], output=[k#, sum#])\n         +- LocalTableScan [k#, v#]\n"]]
diff --git a/Tests/SparkConnectTests/Resources/queries/show_databases.sql b/Tests/SparkConnectTests/Resources/queries/show_databases.sql
@@ -0,0 +1 @@
+SHOW DATABASES
diff --git a/Tests/SparkConnectTests/Resources/queries/show_databases.sql.json b/Tests/SparkConnectTests/Resources/queries/show_databases.sql.json
@@ -0,0 +1 @@
+[["default"]]
diff --git a/Tests/SparkConnectTests/Resources/queries/show_tables.sql b/Tests/SparkConnectTests/Resources/queries/show_tables.sql
@@ -0,0 +1 @@
+SHOW TABLES
diff --git a/Tests/SparkConnectTests/Resources/queries/show_tables.sql.json b/Tests/SparkConnectTests/Resources/queries/show_tables.sql.json
@@ -0,0 +1 @@
+[["","testcache","true"]]
diff --git a/Tests/SparkConnectTests/SQLTests.swift b/Tests/SparkConnectTests/SQLTests.swift
@@ -28,6 +28,22 @@ struct SQLTests {
   let path = Bundle.module.path(forResource: "queries", ofType: "")!
   let encoder = JSONEncoder()
 
+  let regexID = /#\d+L?/
+  let regexPlanId = /plan_id=\d+/
+
+  private func removeID(_ str: String) -> String {
+    return str.replacing(regexPlanId, with: "plan_id=").replacing(regexID, with: "#")
+  }
+
+  @Test
+  func testRemoveID() {
+    #expect(removeID("123") == "123")
+    #expect(removeID("123L") == "123L")
+    #expect(removeID("#123") == "#")
+    #expect(removeID("#123L") == "#")
+    #expect(removeID("plan_id=123") == "plan_id=")
+  }
+
 #if !os(Linux)
   @Test
   func runAll() async throws {
@@ -38,8 +54,8 @@ struct SQLTests {
 
       let sql = try String(contentsOf: URL(fileURLWithPath: "\(path)/\(name)"), encoding: .utf8)
       let jsonData = try encoder.encode(try await spark.sql(sql).collect())
-      let answer = String(data: jsonData, encoding: .utf8)!
-      let expected = try String(contentsOf: URL(fileURLWithPath: "\(path)/\(name).json"), encoding: .utf8)
+      let answer = removeID(String(data: jsonData, encoding: .utf8)!)
+      let expected = removeID(try String(contentsOf: URL(fileURLWithPath: "\(path)/\(name).json"), encoding: .utf8))
       #expect(answer == expected.trimmingCharacters(in: .whitespacesAndNewlines))
     }
     await spark.stop()

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+[["Catalog Name","spark_catalog"],["Namespace Name","default"],["Comment","default database"],["Location","file:\/opt\/spark\/work-dir\/spark-warehouse"],["Owner","185"]]`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+[["== Parsed Logical Plan ==\n'Aggregate ['k], ['k, unresolvedalias('sum('v))]\n+- SubqueryAlias t\n +- LocalRelation [k#, v#]\n\n== Analyzed Logical Plan ==\nk: int, sum(v): bigint\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- SubqueryAlias t\n +- LocalRelation [k#, v#]\n\n== Optimized Logical Plan ==\nAggregate [k#], [k#, sum(v#) AS sum(v)#]\n+- LocalRelation [k#, v#]\n\n== Physical Plan ==\nAdaptiveSparkPlan isFinalPlan=false\n+- HashAggregate(keys=[k#], functions=[sum(v#)], output=[k#, sum(v)#])\n +- Exchange hashpartitioning(k#, 200), ENSURE_REQUIREMENTS, [plan_id=]\n +- HashAggregate(keys=[k#], functions=[partial_sum(v#)], output=[k#, sum#])\n +- LocalTableScan [k#, v#]\n"]]