Merge #152685

152685: sql: include unoptimized right side into apply join EXPLAIN r=yuzefovich a=yuzefovich

The apply join operator works by creating an optimized right side plan for each input row. Previously, this right side was completely omitted from the EXPLAIN outputs. This commit makes things a bit better by including the stringified form of the unoptimized right side plan (similar to the one that can be viewed with `EXPLAIN (OPT)`) in all EXPLAIN variants. In EXPLAIN this looks ok but in EXPLAIN ANALYZE, since the new addition is not annotated with execution statistics, it looks a bit out of place, but I think it's still better than nothing - at least it shows that there is something else going on within apply join that was previously completely hidden.

Fixes: #89585.
Release note: None

Co-authored-by: Yahor Yuzefovich <[email protected]>

Author: craig[bot]

pkg/sql/distsql_spec_exec_factory.go

@@ -510,6 +510,7 @@ func (e *distSQLSpecExecFactory) ConstructApplyJoin(
 	rightColumns colinfo.ResultColumns,
 	onCond tree.TypedExpr,
 	planRightSideFn exec.ApplyJoinPlanRightSideFn,
+	rightSideForExplainFn exec.ApplyJoinRightSideForExplainFn,
 ) (exec.Node, error) {
 	return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: apply join")
 }

pkg/sql/opt/exec/execbuilder/relational.go

@@ -1349,6 +1349,16 @@ func (b *Builder) buildApplyJoin(join memo.RelExpr) (_ execPlan, outputCols colO
 		return plan, nil
 	}
 
+	// Build the stringified representation of the unoptimized right side for
+	// EXPLAIN purposes on demand.
+	rightSideForExplainFn := func(redactableValues bool) string {
+		f := memo.MakeExprFmtCtx(
+			b.ctx, memo.ExprFmtHideAll, redactableValues, b.mem, b.catalog,
+		)
+		f.FormatExpr(rightExpr)
+		return f.Buffer.String()
+	}
+
 	// The right plan will always produce the columns in the presentation, in
 	// the same order. This map is only used for the lifetime of this function,
 	// so free the map afterward.
@@ -1391,6 +1401,7 @@ func (b *Builder) buildApplyJoin(join memo.RelExpr) (_ execPlan, outputCols colO
 		b.presentationToResultColumns(rightRequiredProps.Presentation),
 		onExpr,
 		planRightSideFn,
+		rightSideForExplainFn,
 	)
 	if err != nil {
 		return execPlan{}, colOrdMap{}, err

pkg/sql/opt/exec/execbuilder/testdata/explain_redact

@@ -2616,10 +2616,24 @@ vectorized: true
     │
     └── • apply join (left outer)
         │
-        └── • scan
-              missing stats
-              table: f@f_pkey
-              spans: FULL SCAN
+        ├── • scan
+        │     missing stats
+        │     table: f@f_pkey
+        │     spans: FULL SCAN
+        │
+        └── • inner loop (unoptimized)
+            │
+            └── • distinct-on
+                   └── project
+                        ├── select
+                        │    ├── scan bc
+                        │    │    └── computed column expressions
+                        │    │         └── c
+                        │    │              └── b * ‹×›
+                        │    └── filters
+                        │         └── (b * f) < ‹×›
+                        └── projections
+                             └── (f + (b * ‹×›)) + ‹×›
 
 query T
 EXPLAIN (VERBOSE, REDACT) SELECT f, g FROM f, LATERAL (SELECT count(DISTINCT c + f + 1) * 2 AS g FROM bc WHERE b * f < 10)
@@ -2643,11 +2657,27 @@ vectorized: true
         │ columns: (f, rowid, column13)
         │ estimated row count: 333,333 (missing stats)
         │
-        └── • scan
-              columns: (f, rowid)
-              estimated row count: 1,000 (missing stats)
-              table: f@f_pkey
-              spans: FULL SCAN
+        ├── • scan
+        │     columns: (f, rowid)
+        │     estimated row count: 1,000 (missing stats)
+        │     table: f@f_pkey
+        │     spans: FULL SCAN
+        │
+        └── • inner loop (unoptimized)
+            │ columns: ()
+            │
+            └── • distinct-on
+                  columns: ()
+                   └── project
+                        ├── select
+                        │    ├── scan bc
+                        │    │    └── computed column expressions
+                        │    │         └── c
+                        │    │              └── b * ‹×›
+                        │    └── filters
+                        │         └── (b * f) < ‹×›
+                        └── projections
+                             └── (f + (b * ‹×›)) + ‹×›
 
 query T
 EXPLAIN (OPT, REDACT) SELECT f, g FROM f, LATERAL (SELECT count(DISTINCT c + f + 1) * 2 AS g FROM bc WHERE b * f < 10)
@@ -3022,10 +3052,23 @@ vectorized: true
 • apply join (anti)
 │ pred: (a <= "?column?") IS NOT ‹×›
 │
-└── • scan
-      missing stats
-      table: a@a_pkey
-      spans: FULL SCAN
+├── • scan
+│     missing stats
+│     table: a@a_pkey
+│     spans: FULL SCAN
+│
+└── • inner loop (unoptimized)
+    │
+    └── • project
+           ├── select
+           │    ├── scan bc
+           │    │    └── computed column expressions
+           │    │         └── c
+           │    │              └── b * ‹×›
+           │    └── filters
+           │         └── b > (a::FLOAT8 * ‹×›)
+           └── projections
+                └── (b * ‹×›)::INT8 + ‹×›
 
 query T
 EXPLAIN (VERBOSE, REDACT) SELECT * FROM a WHERE a > ALL (SELECT c::int + 2 FROM bc WHERE b > a::float * 3)
@@ -3038,11 +3081,26 @@ vectorized: true
 │ estimated row count: 667 (missing stats)
 │ pred: (a <= "?column?") IS NOT ‹×›
 │
-└── • scan
-      columns: (a)
-      estimated row count: 1,000 (missing stats)
-      table: a@a_pkey
-      spans: FULL SCAN
+├── • scan
+│     columns: (a)
+│     estimated row count: 1,000 (missing stats)
+│     table: a@a_pkey
+│     spans: FULL SCAN
+│
+└── • inner loop (unoptimized)
+    │ columns: ()
+    │
+    └── • project
+          columns: ()
+           ├── select
+           │    ├── scan bc
+           │    │    └── computed column expressions
+           │    │         └── c
+           │    │              └── b * ‹×›
+           │    └── filters
+           │         └── b > (a::FLOAT8 * ‹×›)
+           └── projections
+                └── (b * ‹×›)::INT8 + ‹×›
 
 query T
 EXPLAIN (OPT, REDACT) SELECT * FROM a WHERE a > ALL (SELECT c::int + 2 FROM bc WHERE b > a::float * 3)

pkg/sql/opt/exec/execbuilder/testdata/srfs

@@ -303,11 +303,19 @@ vectorized: true
                 │ columns: (x, y, unnest)
                 │ estimated row count: 2,000 (missing stats)
                 │
-                └── • scan
-                      columns: (x, y)
-                      estimated row count: 1,000 (missing stats)
-                      table: xy@xy_pkey
-                      spans: FULL SCAN
+                ├── • scan
+                │     columns: (x, y)
+                │     estimated row count: 1,000 (missing stats)
+                │     table: xy@xy_pkey
+                │     spans: FULL SCAN
+                │
+                └── • inner loop (unoptimized)
+                    │ columns: ()
+                    │
+                    └── • values
+                          columns: ()
+                           ├── (xy.x,)
+                           └── (y,)
 
 # Regression test for #24676.
 statement ok

pkg/sql/opt/exec/execbuilder/testdata/subquery

@@ -437,11 +437,19 @@ vectorized: true
 │ estimated row count: 2 (missing stats)
 │ pred: column1 = a
 │
-└── • scan
-      columns: (a, b, c)
-      estimated row count: 1,000 (missing stats)
-      table: abc@abc_pkey
-      spans: FULL SCAN
+├── • scan
+│     columns: (a, b, c)
+│     estimated row count: 1,000 (missing stats)
+│     table: abc@abc_pkey
+│     spans: FULL SCAN
+│
+└── • inner loop (unoptimized)
+    │ columns: ()
+    │
+    └── • values
+          columns: ()
+           ├── (a,)
+           └── (b,)
 
 statement ok
 CREATE TABLE corr (

pkg/sql/opt/exec/execbuilder/testdata/with

@@ -213,10 +213,20 @@ vectorized: true
     │
     └── • apply join (left outer)
         │
-        └── • scan
-              missing stats
-              table: y@y_pkey
-              spans: FULL SCAN
+        ├── • scan
+        │     missing stats
+        │     table: y@y_pkey
+        │     spans: FULL SCAN
+        │
+        └── • inner loop (unoptimized)
+            │
+            └── • with &1 (foo)
+                   ├── materialized
+                   ├── select
+                   │    ├── scan x
+                   │    └── filters
+                   │         └── x.a = y.a
+                   └── with-scan &1 (foo)
 
 query T
 EXPLAIN
@@ -229,8 +239,18 @@ vectorized: true
 ·
 • apply join
 │
-└── • values
-      size: 1 column, 3 rows
+├── • values
+│     size: 1 column, 3 rows
+│
+└── • inner loop (unoptimized)
+    │
+    └── • with &1 (foo)
+           ├── materialized
+           ├── select
+           │    ├── scan y
+           │    └── filters
+           │         └── y.a <= column1
+           └── with-scan &1 (foo)
 
 # Regression tests for #93370. Do not convert a non-recursive CTE
 # that uses UNION ALL and WITH RECURSIVE to UNION.

pkg/sql/opt/exec/explain/emit.go

@@ -99,6 +99,7 @@ func emitInternal(
 				return err
 			}
 		}
+		e.emitNodeFootnotes(n)
 		ob.LeaveNode()
 		return nil
 	}
@@ -1269,6 +1270,35 @@ func (e *emitter) emitNodeAttributes(ctx context.Context, evalCtx *eval.Context,
 	return nil
 }
 
+// emitNodeFootnotes populates the Node's information after having recursed into
+// the Node's children.
+func (e *emitter) emitNodeFootnotes(n *Node) {
+	switch n.op {
+	case applyJoinOp:
+		a := n.args.(*applyJoinArgs)
+		if a.RightSideForExplainFn != nil {
+			e.ob.EnterNode("inner loop (unoptimized)" /* name */, nil /* columns */, nil /* ordering */)
+			defer e.ob.LeaveNode()
+			// RightSideForExplainFn can produce multiple lines that correspond
+			// to the unoptimized right side plan.
+			lines := strings.Split(a.RightSideForExplainFn(e.ob.flags.RedactValues), "\n")
+			var enteredNode bool
+			for _, l := range lines {
+				if len(l) == 0 {
+					continue
+				}
+				if !enteredNode {
+					e.ob.EnterNode(l /* name */, nil /* columns */, nil /* ordering */)
+					defer e.ob.LeaveNode() //nolint:deferloop
+					enteredNode = true
+				} else {
+					e.ob.Attr(l, "")
+				}
+			}
+		}
+	}
+}
+
 func (e *emitter) emitTableAndIndex(field string, table cat.Table, index cat.Index, suffix string) {
 	partial := ""
 	if _, isPartial := index.Predicate(); isPartial {

pkg/sql/opt/exec/explain/testdata/gists

@@ -132,9 +132,15 @@ explain(shape):
 • apply join (semi)
 │ pred: column1 = a
 │
-└── • scan
-      table: abc@abc_pkey
-      spans: FULL SCAN
+├── • scan
+│     table: abc@abc_pkey
+│     spans: FULL SCAN
+│
+└── • inner loop (unoptimized)
+    │
+    └── • values
+           ├── (a,)
+           └── (b,)
 explain(gist):
 • apply join (semi)
 │

pkg/sql/opt/exec/factory.go

@@ -292,6 +292,11 @@ type RecursiveCTEIterationFn func(ctx context.Context, ef Factory, bufferRef Nod
 // rightColumns passed to ConstructApplyJoin (in order).
 type ApplyJoinPlanRightSideFn func(ctx context.Context, ef Factory, leftRow tree.Datums) (Plan, error)
 
+// ApplyJoinRightSideForExplainFn is a function that lazily populates the
+// stringified version of the unoptimized right-hand side plan, for EXPLAIN
+// purposes.
+type ApplyJoinRightSideForExplainFn func(redactableValues bool) string
+
 // PostQuery describes a cascading query or an AFTER trigger action. The query
 // uses a node created by ConstructBuffer as an input; it should only be
 // triggered if this buffer is not empty.

pkg/sql/opt/exec/factory.opt

@@ -82,6 +82,7 @@ define ApplyJoin {
     RightColumns colinfo.ResultColumns
     OnCond tree.TypedExpr
     PlanRightSideFn exec.ApplyJoinPlanRightSideFn
+    RightSideForExplainFn exec.ApplyJoinRightSideForExplainFn
 }
 
 # HashJoin runs a hash-join between the results of two input nodes.