Merge #143733

143733: opt: do not hoist outer columns from the right input of a left-join r=mgartner a=DrewKimball

Recently in #142251 we improved the `HoistJoinProjectRight` norm rule to allow hoisting a `Project` with remapping projections, which simply map from one column ID to another. However, there is an issue when the remapped column is an outer column and the join is a left-join: after the `Project` is hoisted, the projection will no longer be `NULL` for rows that were null-extended by the join. This causes incorrect results. The fix is simply to check that only input columns are remapped, since they will properly reflect the effects of null-extension.

There is no release note because the rule change with the bug is only on master.

Fixes #142531

Release note: None

Co-authored-by: Drew Kimball <[email protected]>

Author: craig[bot]

pkg/sql/opt/exec/execbuilder/testdata/vectorize_local

@@ -499,8 +499,9 @@ EXPLAIN (VEC)
 └ Node 1
   └ *colexec.caseOp
     ├ *colexec.bufferOp
-    │ └ *colfetcher.ColIndexJoin
-    │   └ *colfetcher.ColBatchScan
+    │ └ *sql.planNodeToRowSource
+    │   └ *colfetcher.ColIndexJoin
+    │     └ *colfetcher.ColBatchScan
     ├ *colexec.bufferOp
     └ *colexec.bufferOp
 

pkg/sql/opt/memo/testdata/logprops/join

@@ -3352,10 +3352,85 @@ SELECT (
 FROM table3_119441 AS baz;
 ----
 project
- ├── columns: col3_5:28(varchar!null)
+ ├── columns: col3_5:28(varchar)
  ├── prune: (28)
- ├── scan table3_119441 [as=baz]
- │    ├── columns: baz.col3_5:1(varchar!null)
- │    └── prune: (1)
+ ├── reject-nulls: (28)
+ ├── left-join-apply
+ │    ├── columns: baz.col3_5:1(varchar!null) baz.col3_11:2(char) col3_5:27(varchar)
+ │    ├── prune: (27)
+ │    ├── reject-nulls: (27)
+ │    ├── scan table3_119441 [as=baz]
+ │    │    ├── columns: baz.col3_5:1(varchar!null) baz.col3_11:2(char)
+ │    │    ├── prune: (1,2)
+ │    │    └── unfiltered-cols: (1-6)
+ │    ├── project
+ │    │    ├── columns: col3_5:27(varchar)
+ │    │    ├── outer: (1,2)
+ │    │    ├── cardinality: [0 - 1]
+ │    │    ├── key: ()
+ │    │    ├── fd: ()-->(27)
+ │    │    ├── prune: (27)
+ │    │    ├── limit
+ │    │    │    ├── columns: col1_8:8(bytes!null) col1_11:9(string!null) col1_12:10(string!null) col2_1:14(int4!null) col2_2:15(string!null) col2_3:16(bytes!null) table3_119441.col3_16:22(int!null)
+ │    │    │    ├── outer: (2)
+ │    │    │    ├── cardinality: [0 - 1]
+ │    │    │    ├── key: ()
+ │    │    │    ├── fd: ()-->(8-10,14-16,22), (9)==(10,15), (10)==(9,15), (15)==(9,10), (8)==(16), (16)==(8), (14)==(22), (22)==(14)
+ │    │    │    ├── interesting orderings: (+(9|10)) (+22)
+ │    │    │    ├── inner-join (hash)
+ │    │    │    │    ├── columns: col1_8:8(bytes!null) col1_11:9(string!null) col1_12:10(string!null) col2_1:14(int4!null) col2_2:15(string!null) col2_3:16(bytes!null) table3_119441.col3_16:22(int!null)
+ │    │    │    │    ├── outer: (2)
+ │    │    │    │    ├── fd: ()-->(9,10,15), (9)==(10,15), (10)==(9,15), (15)==(9,10), (8)==(16), (16)==(8), (14)==(22), (22)==(14)
+ │    │    │    │    ├── limit hint: 1.00
+ │    │    │    │    ├── interesting orderings: (+(9|10)) (+22)
+ │    │    │    │    ├── top-k
+ │    │    │    │    │    ├── columns: table3_119441.col3_16:22(int)
+ │    │    │    │    │    ├── internal-ordering: +22
+ │    │    │    │    │    ├── k: 6
+ │    │    │    │    │    ├── cardinality: [0 - 6]
+ │    │    │    │    │    ├── interesting orderings: (+22)
+ │    │    │    │    │    └── scan table3_119441
+ │    │    │    │    │         ├── columns: table3_119441.col3_16:22(int)
+ │    │    │    │    │         └── prune: (22)
+ │    │    │    │    ├── inner-join (hash)
+ │    │    │    │    │    ├── columns: col1_8:8(bytes!null) col1_11:9(string!null) col1_12:10(string!null) col2_1:14(int4) col2_2:15(string!null) col2_3:16(bytes!null)
+ │    │    │    │    │    ├── fd: (9)==(10,15), (10)==(9,15), (15)==(9,10), (8)==(16), (16)==(8)
+ │    │    │    │    │    ├── prune: (14)
+ │    │    │    │    │    ├── interesting orderings: (+(9|10))
+ │    │    │    │    │    ├── scan table2_119441
+ │    │    │    │    │    │    ├── columns: col2_1:14(int4) col2_2:15(string) col2_3:16(bytes)
+ │    │    │    │    │    │    ├── prune: (14-16)
+ │    │    │    │    │    │    └── unfiltered-cols: (14-19)
+ │    │    │    │    │    ├── select
+ │    │    │    │    │    │    ├── columns: col1_8:8(bytes) col1_11:9(string!null) col1_12:10(string!null)
+ │    │    │    │    │    │    ├── fd: (9)==(10), (10)==(9)
+ │    │    │    │    │    │    ├── prune: (8)
+ │    │    │    │    │    │    ├── interesting orderings: (+(9|10))
+ │    │    │    │    │    │    ├── scan table1_119441
+ │    │    │    │    │    │    │    ├── columns: col1_8:8(bytes) col1_11:9(string) col1_12:10(string)
+ │    │    │    │    │    │    │    ├── prune: (8-10)
+ │    │    │    │    │    │    │    └── interesting orderings: (+9)
+ │    │    │    │    │    │    └── filters
+ │    │    │    │    │    │         └── eq [type=bool, outer=(9,10), constraints=(/9: (/NULL - ]; /10: (/NULL - ]), fd=(9)==(10), (10)==(9)]
+ │    │    │    │    │    │              ├── variable: col1_11:9 [type=string]
+ │    │    │    │    │    │              └── variable: col1_12:10 [type=string]
+ │    │    │    │    │    └── filters
+ │    │    │    │    │         ├── eq [type=bool, outer=(8,16), constraints=(/8: (/NULL - ]; /16: (/NULL - ]), fd=(8)==(16), (16)==(8)]
+ │    │    │    │    │         │    ├── variable: col1_8:8 [type=bytes]
+ │    │    │    │    │         │    └── variable: col2_3:16 [type=bytes]
+ │    │    │    │    │         └── eq [type=bool, outer=(9,15), constraints=(/9: (/NULL - ]; /15: (/NULL - ]), fd=(9)==(15), (15)==(9)]
+ │    │    │    │    │              ├── variable: col1_11:9 [type=string]
+ │    │    │    │    │              └── variable: col2_2:15 [type=string]
+ │    │    │    │    └── filters
+ │    │    │    │         ├── eq [type=bool, outer=(14,22), constraints=(/14: (/NULL - ]; /22: (/NULL - ]), fd=(14)==(22), (22)==(14)]
+ │    │    │    │         │    ├── variable: col2_1:14 [type=int4]
+ │    │    │    │         │    └── variable: table3_119441.col3_16:22 [type=int]
+ │    │    │    │         └── eq [type=bool, outer=(2,9), constraints=(/2: (/NULL - ]; /9: (/NULL - ]), fd=(2)==(9), (9)==(2)]
+ │    │    │    │              ├── variable: col1_11:9 [type=string]
+ │    │    │    │              └── variable: baz.col3_11:2 [type=char]
+ │    │    │    └── const: 1 [type=int]
+ │    │    └── projections
+ │    │         └── variable: baz.col3_5:1 [as=col3_5:27, type=varchar, outer=(1)]
+ │    └── filters (true)
  └── projections
-      └── variable: baz.col3_5:1 [as=col3_5:28, type=varchar, outer=(1)]
+      └── variable: col3_5:27 [as=col3_5:28, type=varchar, outer=(27)]

pkg/sql/opt/norm/rules/join.opt

@@ -427,7 +427,16 @@ $left
     $right:(Project
         $input:*
         $projections:* &
-            (AllAreRemappingProjections $projections)
+            (AllAreRemappingProjections $projections) &
+
+            # Ensure that there are no outer-column references in the
+            # projections, since otherwise hoisting the Project could change
+            # the result of a left-join due to the NULL-extended rows.
+            # TODO(drewk): we could allow this for inner-joins.
+            (ColsAreSubset
+                (ProjectionOuterCols $projections)
+                (OutputCols $input)
+            )
         $passThrough:*
     )
     $on:*

pkg/sql/opt/norm/testdata/rules/join

@@ -2878,6 +2878,56 @@ project
  └── projections
       └── c.y:6 [as=k:10, outer=(6)]
 
+# Regression test for #142531 - do not hoist outer-column references from the
+# right input of a left-join.
+exec-ddl
+CREATE TABLE table_142531 (col1_1 INT NOT NULL);
+----
+
+norm expect-not=HoistJoinProjectRight
+SELECT (SELECT t.col1_1 FROM table_142531 WHERE (NOT EXISTS (SELECT (0)))) FROM table_142531 AS t;
+----
+project
+ ├── columns: col1_1:13
+ ├── ensure-distinct-on
+ │    ├── columns: t.rowid:2!null col1_1:12
+ │    ├── grouping columns: t.rowid:2!null
+ │    ├── error: "more than one row returned by a subquery used as an expression"
+ │    ├── key: (2)
+ │    ├── fd: (2)-->(12)
+ │    ├── left-join-apply
+ │    │    ├── columns: t.col1_1:1!null t.rowid:2!null col1_1:12
+ │    │    ├── fd: (2)-->(1)
+ │    │    ├── scan table_142531 [as=t]
+ │    │    │    ├── columns: t.col1_1:1!null t.rowid:2!null
+ │    │    │    ├── key: (2)
+ │    │    │    └── fd: (2)-->(1)
+ │    │    ├── project
+ │    │    │    ├── columns: col1_1:12
+ │    │    │    ├── outer: (1)
+ │    │    │    ├── fd: ()-->(12)
+ │    │    │    ├── select
+ │    │    │    │    ├── scan table_142531
+ │    │    │    │    └── filters
+ │    │    │    │         └── not [subquery]
+ │    │    │    │              └── coalesce
+ │    │    │    │                   ├── subquery
+ │    │    │    │                   │    └── values
+ │    │    │    │                   │         ├── columns: column11:11!null
+ │    │    │    │                   │         ├── cardinality: [1 - 1]
+ │    │    │    │                   │         ├── key: ()
+ │    │    │    │                   │         ├── fd: ()-->(11)
+ │    │    │    │                   │         └── (true,)
+ │    │    │    │                   └── false
+ │    │    │    └── projections
+ │    │    │         └── t.col1_1:1 [as=col1_1:12, outer=(1)]
+ │    │    └── filters (true)
+ │    └── aggregations
+ │         └── const-agg [as=col1_1:12, outer=(12)]
+ │              └── col1_1:12
+ └── projections
+      └── col1_1:12 [as=col1_1:13, outer=(12)]
+
 # --------------------------------------------------
 # HoistJoinProjectLeft
 # --------------------------------------------------

pkg/sql/opt/norm/testdata/rules/reject_nulls

@@ -1673,21 +1673,22 @@ project
  ├── sort
  │    ├── columns: tab_3663.col2_3:2 tab_3663.col2_5:4 tab_3663.col2_6:5!null tab_3663.col2_9:6 tab_3663.crdb_internal_mvcc_timestamp:11 tab_3663.tableoid:12
  │    ├── ordering: -11,-12,+4
- │    └── semi-join (hash)
+ │    └── semi-join-apply
  │         ├── columns: tab_3663.col2_3:2 tab_3663.col2_5:4 tab_3663.col2_6:5!null tab_3663.col2_9:6 tab_3663.crdb_internal_mvcc_timestamp:11 tab_3663.tableoid:12
  │         ├── scan table2_89986 [as=tab_3663]
  │         │    └── columns: tab_3663.col2_3:2 tab_3663.col2_5:4 tab_3663.col2_6:5!null tab_3663.col2_9:6 tab_3663.crdb_internal_mvcc_timestamp:11 tab_3663.tableoid:12
- │         ├── inner-join (cross)
- │         │    ├── columns: tab_3664.col2_12:21!null col1_3:28!null col1_8:33!null tab_3666.col2_9:43 tab_3666.col2_11:45!null
- │         │    ├── fd: (33)==(45), (45)==(33), (21)==(28), (28)==(21)
+ │         ├── inner-join (hash)
+ │         │    ├── columns: tab_3664.col2_12:21!null col1_3:28!null col1_8:33 tab_3666.col2_9:43!null tab_3666.col2_11:45 col2_9:62!null
+ │         │    ├── outer: (6)
+ │         │    ├── fd: ()-->(43,62), (43)==(62), (62)==(43), (21)==(28), (28)==(21)
+ │         │    ├── scan table2_89986 [as=tab_3664]
+ │         │    │    └── columns: tab_3664.col2_12:21
  │         │    ├── inner-join (hash)
- │         │    │    ├── columns: tab_3664.col2_12:21!null col1_3:28!null col1_8:33!null tab_3666.col2_9:43 tab_3666.col2_11:45!null
- │         │    │    ├── fd: (33)==(45), (45)==(33), (21)==(28), (28)==(21)
- │         │    │    ├── scan table2_89986 [as=tab_3664]
- │         │    │    │    └── columns: tab_3664.col2_12:21
- │         │    │    ├── inner-join (hash)
- │         │    │    │    ├── columns: col1_3:28!null col1_8:33!null tab_3666.col2_9:43 tab_3666.col2_11:45!null
- │         │    │    │    ├── fd: (33)==(45), (45)==(33)
+ │         │    │    ├── columns: col1_3:28 col1_8:33 tab_3666.col2_9:43!null tab_3666.col2_11:45 col2_9:62!null
+ │         │    │    ├── outer: (6)
+ │         │    │    ├── fd: ()-->(43,62), (43)==(62), (62)==(43)
+ │         │    │    ├── left-join (hash)
+ │         │    │    │    ├── columns: col1_3:28 col1_8:33 tab_3666.col2_9:43 tab_3666.col2_11:45
  │         │    │    │    ├── scan table2_89986 [as=tab_3666]
  │         │    │    │    │    └── columns: tab_3666.col2_9:43 tab_3666.col2_11:45
  │         │    │    │    ├── scan table1_89986
@@ -1699,12 +1700,18 @@ project
  │         │    │    │    │              └── col1_0:25 + col1_1:26
  │         │    │    │    └── filters
  │         │    │    │         └── col1_8:33 = tab_3666.col2_11:45 [outer=(33,45), constraints=(/33: (/NULL - ]; /45: (/NULL - ]), fd=(33)==(45), (45)==(33)]
+ │         │    │    ├── project
+ │         │    │    │    ├── columns: col2_9:62
+ │         │    │    │    ├── outer: (6)
+ │         │    │    │    ├── fd: ()-->(62)
+ │         │    │    │    ├── scan table2_89986 [as=tab_3667]
+ │         │    │    │    └── projections
+ │         │    │    │         └── tab_3663.col2_9:6 [as=col2_9:62, outer=(6)]
  │         │    │    └── filters
- │         │    │         └── tab_3664.col2_12:21 = col1_3:28 [outer=(21,28), constraints=(/21: (/NULL - ]; /28: (/NULL - ]), fd=(21)==(28), (28)==(21)]
- │         │    ├── scan table2_89986 [as=tab_3667]
- │         │    └── filters (true)
- │         └── filters
- │              └── tab_3666.col2_9:43 = tab_3663.col2_9:6 [outer=(6,43), constraints=(/6: (/NULL - ]; /43: (/NULL - ]), fd=(6)==(43), (43)==(6)]
+ │         │    │         └── tab_3666.col2_9:43 = col2_9:62 [outer=(43,62), constraints=(/43: (/NULL - ]; /62: (/NULL - ]), fd=(43)==(62), (62)==(43)]
+ │         │    └── filters
+ │         │         └── tab_3664.col2_12:21 = col1_3:28 [outer=(21,28), constraints=(/21: (/NULL - ]; /28: (/NULL - ]), fd=(21)==(28), (28)==(21)]
+ │         └── filters (true)
  └── projections
       ├── 0 [as="?column?":67]
       ├── '23:43:20-08:00:00' [as="?column?":68]