Merge #146494

146494: opt: simplify x is not distinct from y to x = y r=mgartner a=sgasho

This commit adds a new select rule SimplifyIsCondition which rewrites
expressions in the form `x IS NOT DISTINCT FROM y` to `x = y`. This
transformation is only valid if all of the following are true:

  1. The expression is in the context of filtering where NULL is falsy.
  2. One of x or y is non-nullable. This is required because while the
     expression NULL IS NOT DISTINCT FROM NULL is true, NULL=NULL is NULL
     (falsy).
  3. Neither x nor y is a tuple. Tuples with NULLs have all sorts of
     complicated edge cases, so we avoid them entirely. See #48299.

We conservatively also require the types of x and y to be identical. It may be
possible to lift this restriction if we can prove that it is not necessary.

Fixes #144524

Release note (performance improvement): Some queries with filters in the
form `x IS NOT DISTINCT FROM y` now have more optimal query plans.

Co-authored-by: sgasho <[email protected]>

Author: craig[bot]

pkg/sql/opt/norm/general_funcs.go

@@ -71,6 +71,11 @@ func (c *CustomFuncs) IsInt(scalar opt.ScalarExpr) bool {
 	return scalar.DataType().Family() == types.IntFamily
 }
 
+// IsTuple returns true if the given scalar expression is a tuple type.
+func (c *CustomFuncs) IsTuple(scalar opt.ScalarExpr) bool {
+	return scalar.DataType().Family() == types.TupleFamily
+}
+
 // BoolType returns the boolean SQL type.
 func (c *CustomFuncs) BoolType() *types.T {
 	return types.Bool
@@ -107,6 +112,12 @@ func (c *CustomFuncs) TypeOf(e opt.ScalarExpr) *types.T {
 	return e.DataType()
 }
 
+// IdenticalTypes returns true if the two types are identical. See
+// (*types.T).Identical.
+func (c *CustomFuncs) IdenticalTypes(left, right *types.T) bool {
+	return left.Identical(right)
+}
+
 // IsConstArray returns true if the expression is a constant array.
 func (c *CustomFuncs) IsConstArray(scalar opt.ScalarExpr) bool {
 	if cnst, ok := scalar.(*memo.ConstExpr); ok {
@@ -614,6 +625,13 @@ func (c *CustomFuncs) ExprIsNeverNull(e opt.ScalarExpr, notNullCols opt.ColSet)
 	return memo.ExprIsNeverNull(e, notNullCols)
 }
 
+// EitherExprIsNeverNull returns true if either of the two provided scalar
+// expressions is guaranteed to be non-NULL, given the set of outer columns that
+// are known to be not null.
+func (c *CustomFuncs) EitherExprIsNeverNull(a, b opt.ScalarExpr, notNullCols opt.ColSet) bool {
+	return memo.ExprIsNeverNull(a, notNullCols) || memo.ExprIsNeverNull(b, notNullCols)
+}
+
 // sharedProps returns the shared logical properties for the given expression.
 // Only relational expressions and certain scalar list items (e.g. FiltersItem,
 // ProjectionsItem, AggregationsItem) have shared properties.

pkg/sql/opt/norm/rules/select.opt

@@ -354,6 +354,48 @@ $input
 =>
 (Select $input [ (FiltersItem (False)) ])
 
+# SimplifyIsCondition replaces x IS NOT DISTINCT FROM y with x = y. This
+# transformation is only valid if all of the following are true:
+#
+#   1. The expression is in the context of filtering where NULL is falsy.
+#   2. One of x or y is non-nullable. This is required because while the
+#      expression NULL IS NOT DISTINCT FROM NULL is true, NULL=NULL is NULL
+#      (falsy).
+#   3. Neither x nor y is a tuple. Tuples with NULLs have all sorts of
+#      complicated edge cases, so we avoid them entirely. See #48299.
+#
+# We conservatively also require the types of x and y to be identical. It may be
+# possible to lift this restriction if we can prove that it is not necessary.
+[SimplifyIsCondition, Normalize]
+(Select
+    $input:*
+    $filters:[
+        ...
+        $item:(FiltersItem
+            (Is
+                $left:* & ^(IsTuple $left)
+                $right:* &
+                    ^(IsTuple $right) &
+                    (IdenticalTypes
+                        (TypeOf $left)
+                        (TypeOf $right)
+                    ) &
+                    (EitherExprIsNeverNull
+                        $left
+                        $right
+                        (NotNullCols $input)
+                    )
+            )
+        )
+        ...
+    ]
+)
+=>
+(Select
+    $input
+    (ReplaceFiltersItem $filters $item (Eq $left $right))
+)
+
 # PushSelectIntoProjectSet pushes filters into a ProjectSet. In particular,
 # the filters that are bound to the input columns of the ProjectSet are
 # pushed down into it, in hopes of being pushed down further into joins

pkg/sql/opt/norm/testdata/rules/select

@@ -14,6 +14,10 @@ exec-ddl
 CREATE TABLE c (a BOOL, b BOOL, c BOOL, d BOOL, e BOOL)
 ----
 
+exec-ddl
+CREATE TABLE d (k INT PRIMARY KEY, a INT NOT NULL, b INT, c INT, d FLOAT)
+----
+
 exec-ddl
 CREATE TABLE e
 (
@@ -1638,6 +1642,144 @@ exec-ddl
 DROP INDEX partial_idx
 ----
 
+# --------------------------------------------------
+# SimplifyIsCondition
+# --------------------------------------------------
+
+norm expect=SimplifyIsCondition
+SELECT * FROM d WHERE a = 1 AND k IS NOT DISTINCT FROM 1 AND d = 1.0
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5!null
+ ├── cardinality: [0 - 1]
+ ├── key: ()
+ ├── fd: ()-->(1-5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      ├── a:2 = 1 [outer=(2), constraints=(/2: [/1 - /1]; tight), fd=()-->(2)]
+      ├── k:1 = 1 [outer=(1), constraints=(/1: [/1 - /1]; tight), fd=()-->(1)]
+      └── d:5 = 1.0 [outer=(5), constraints=(/5: [/1.0 - /1.0]; tight), fd=()-->(5)]
+
+# The rule applies if at least one of the operands is non-nullable.
+norm expect=SimplifyIsCondition
+SELECT * FROM d WHERE a IS NOT DISTINCT FROM b
+----
+select
+ ├── columns: k:1!null a:2!null b:3!null c:4 d:5
+ ├── key: (1)
+ ├── fd: (1)-->(2-5), (2)==(3), (3)==(2)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── a:2 = b:3 [outer=(2,3), constraints=(/2: (/NULL - ]; /3: (/NULL - ]), fd=(2)==(3), (3)==(2)]
+
+# The rule does not apply if both operands are nullable.
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE b IS NOT DISTINCT FROM NULL
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: ()-->(3), (1)-->(2,4,5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── b:3 IS NULL [outer=(3), constraints=(/3: [/NULL - /NULL]; tight), fd=()-->(3)]
+
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE NULL IS NOT DISTINCT FROM b
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: ()-->(3), (1)-->(2,4,5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── b:3 IS NULL [outer=(3), constraints=(/3: [/NULL - /NULL]; tight), fd=()-->(3)]
+
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE b IS NOT DISTINCT FROM c
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: (1)-->(2-5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── b:3 IS NOT DISTINCT FROM c:4 [outer=(3,4)]
+
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE b IS NULL
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: ()-->(3), (1)-->(2,4,5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── b:3 IS NULL [outer=(3), constraints=(/3: [/NULL - /NULL]; tight), fd=()-->(3)]
+
+# The rule does not apply if the operands are tuples.
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE (k, a) IS NOT DISTINCT FROM (a, k)
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── immutable
+ ├── key: (1)
+ ├── fd: (1)-->(2-5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── (k:1, a:2) IS NOT DISTINCT FROM (a:2, k:1) [outer=(1,2), immutable]
+
+# The rule does not apply if the operands do not have identical types.
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE a IS NOT DISTINCT FROM 1.23
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: (1)-->(2-5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── a:2 IS NOT DISTINCT FROM 1.23 [outer=(2)]
+
+norm expect-not=SimplifyIsCondition
+SELECT * FROM d WHERE a IS NOT DISTINCT FROM d
+----
+select
+ ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ ├── key: (1)
+ ├── fd: (1)-->(2-5)
+ ├── scan d
+ │    ├── columns: k:1!null a:2!null b:3 c:4 d:5
+ │    ├── key: (1)
+ │    └── fd: (1)-->(2-5)
+ └── filters
+      └── a:2 IS NOT DISTINCT FROM d:5 [outer=(2,5)]
+
 # --------------------------------------------------
 # PushSelectIntoProjectSet
 # --------------------------------------------------