Skip to content

Commit 3a872b7

Browse files
mihailoale-dbcloud-fan
authored andcommitted
[SPARK-51453][SQL] AssertTrue uses toPrettySQL instead of simpleString
### What changes were proposed in this pull request? Make `AssertTrue` constructor use `toPrettySQL` instead of `simpleString`. ### Why are the changes needed? Because of `simpleString` usage we end up having `ExpressionId`s in expression values (and thus in output schema) which can be queried. This is invalid because the behavior of this feature is non-deterministic (query which passes will fail with a cluster restart). Example: ``` SELECT assert_true(col <= col_2) FROM (VALUES ('2025-03-01', '2025-03-10')) t(col,col_2) ``` Output schema will be: `assert_true((col <= col_2), '(col#1214095 <= col_2#1214096)' is not true!)` ### Does this PR introduce _any_ user-facing change? Some SQL queries are going to fail (but they would fail with every cluster reset, as explained). ### How was this patch tested? Existing tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #50226 from mihailoale-db/asserttruefix. Authored-by: mihailoale-db <mihailo.aleksic@databricks.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent 5df3721 commit 3a872b7

File tree

4 files changed

+11
-12
lines changed

4 files changed

+11
-12
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
2323
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
2424
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
2525
import org.apache.spark.sql.catalyst.trees.TreePattern.{CURRENT_LIKE, TreePattern}
26-
import org.apache.spark.sql.catalyst.util.{MapData, RandomUUIDGenerator}
26+
import org.apache.spark.sql.catalyst.util.{toPrettySQL, MapData, RandomUUIDGenerator}
2727
import org.apache.spark.sql.errors.QueryCompilationErrors
2828
import org.apache.spark.sql.errors.QueryExecutionErrors.raiseError
2929
import org.apache.spark.sql.internal.SQLConf
@@ -179,7 +179,7 @@ case class AssertTrue(left: Expression, right: Expression, replacement: Expressi
179179
}
180180

181181
def this(left: Expression) = {
182-
this(left, Literal(s"'${left.simpleString(SQLConf.get.maxToStringFields)}' is not true!"))
182+
this(left, Literal(s"'${toPrettySQL(left)}' is not true!"))
183183
}
184184

185185
override def parameters: Seq[Expression] = Seq(left, right)

sql/core/src/test/resources/sql-tests/analyzer-results/misc-functions.sql.out

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ Project [typeof(array(1, 2)) AS typeof(array(1, 2))#x, typeof(map(1, 2)) AS type
5050
-- !query
5151
SELECT assert_true(true), assert_true(boolean(1))
5252
-- !query analysis
53-
Project [assert_true(true, 'true' is not true!) AS assert_true(true, 'true' is not true!)#x, assert_true(cast(1 as boolean), 'cast(1 as boolean)' is not true!) AS assert_true(1, 'cast(1 as boolean)' is not true!)#x]
53+
Project [assert_true(true, 'true' is not true!) AS assert_true(true, 'true' is not true!)#x, assert_true(cast(1 as boolean), '1' is not true!) AS assert_true(1, '1' is not true!)#x]
5454
+- OneRowRelation
5555

5656

@@ -64,21 +64,21 @@ Project [assert_true(false, 'false' is not true!) AS assert_true(false, 'false'
6464
-- !query
6565
SELECT assert_true(boolean(0))
6666
-- !query analysis
67-
Project [assert_true(cast(0 as boolean), 'cast(0 as boolean)' is not true!) AS assert_true(0, 'cast(0 as boolean)' is not true!)#x]
67+
Project [assert_true(cast(0 as boolean), '0' is not true!) AS assert_true(0, '0' is not true!)#x]
6868
+- OneRowRelation
6969

7070

7171
-- !query
7272
SELECT assert_true(null)
7373
-- !query analysis
74-
Project [assert_true(null, 'null' is not true!) AS assert_true(NULL, 'null' is not true!)#x]
74+
Project [assert_true(null, 'NULL' is not true!) AS assert_true(NULL, 'NULL' is not true!)#x]
7575
+- OneRowRelation
7676

7777

7878
-- !query
7979
SELECT assert_true(boolean(null))
8080
-- !query analysis
81-
Project [assert_true(cast(null as boolean), 'cast(null as boolean)' is not true!) AS assert_true(NULL, 'cast(null as boolean)' is not true!)#x]
81+
Project [assert_true(cast(null as boolean), 'NULL' is not true!) AS assert_true(NULL, 'NULL' is not true!)#x]
8282
+- OneRowRelation
8383

8484

sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ array<int> map<int,int> struct<a:int,b:string>
5858
-- !query
5959
SELECT assert_true(true), assert_true(boolean(1))
6060
-- !query schema
61-
struct<assert_true(true, 'true' is not true!):void,assert_true(1, 'cast(1 as boolean)' is not true!):void>
61+
struct<assert_true(true, 'true' is not true!):void,assert_true(1, '1' is not true!):void>
6262
-- !query output
6363
NULL NULL
6464

@@ -88,7 +88,7 @@ org.apache.spark.SparkRuntimeException
8888
"errorClass" : "USER_RAISED_EXCEPTION",
8989
"sqlState" : "P0001",
9090
"messageParameters" : {
91-
"errorMessage" : "'cast(0 as boolean)' is not true!"
91+
"errorMessage" : "'0' is not true!"
9292
}
9393
}
9494

@@ -103,7 +103,7 @@ org.apache.spark.SparkRuntimeException
103103
"errorClass" : "USER_RAISED_EXCEPTION",
104104
"sqlState" : "P0001",
105105
"messageParameters" : {
106-
"errorMessage" : "'null' is not true!"
106+
"errorMessage" : "'NULL' is not true!"
107107
}
108108
}
109109

@@ -118,7 +118,7 @@ org.apache.spark.SparkRuntimeException
118118
"errorClass" : "USER_RAISED_EXCEPTION",
119119
"sqlState" : "P0001",
120120
"messageParameters" : {
121-
"errorMessage" : "'cast(null as boolean)' is not true!"
121+
"errorMessage" : "'NULL' is not true!"
122122
}
123123
}
124124

sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2596,8 +2596,7 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
25962596
val e3 = intercept[SparkRuntimeException] {
25972597
intDf.select(assert_true($"a" > $"b")).collect()
25982598
}
2599-
assert(e3.getMessage.matches(
2600-
"\\[USER_RAISED_EXCEPTION\\] '\\(a#\\d+ > b#\\d+\\)' is not true! SQLSTATE: P0001"))
2599+
assert(e3.getMessage.equals("[USER_RAISED_EXCEPTION] '(a > b)' is not true! SQLSTATE: P0001"))
26012600
}
26022601

26032602
test("raise_error") {

0 commit comments

Comments
 (0)