sql/opt: add telemetry counters for selectivity clamping

This commit adds telemetry counters for the new histogram selectivity
clamping behavior, as well as log messages to indicate when the new
behavior applies in a query's trace.

Epic: None

Release note: None

Author: DrewKimball

pkg/sql/opt/memo/memo.go

@@ -239,6 +239,10 @@ type Memo struct {
 	// erring with partially normalized expressions.
 	disableCheckExpr bool
 
+	// optimizationStats tracks decisions made during optimization, for example,
+	// to clamp selectivity estimates to a lower bound.
+	optimizationStats OptimizationStats
+
 	// WARNING: if you add more members, add initialization code in Init (if
 	// reusing allocated data structures is desired).
 }
@@ -701,6 +705,25 @@ func (m *Memo) FormatExpr(expr opt.Expr) string {
 	return f.Buffer.String()
 }
 
+// OptimizationStats surfaces information about choices made during optimization
+// of a query for top-level observability (e.g. metrics, EXPLAIN output).
+type OptimizationStats struct {
+	// ClampedHistogramSelectivity is true if the selectivity estimate based on a
+	// histogram was prevented from dropping too low. See also the session var
+	// "optimizer_clamp_low_histogram_selectivity".
+	ClampedHistogramSelectivity bool
+	// ClampedInequalitySelectivity is true if the selectivity estimate for an
+	// inequality unbounded on one or both sides was prevented from dropping too
+	// low. See also the session var "optimizer_clamp_inequality_selectivity".
+	ClampedInequalitySelectivity bool
+}
+
+// GetOptimizationStats returns the OptimizationStats collected during a
+// previous optimization pass.
+func (m *Memo) GetOptimizationStats() *OptimizationStats {
+	return &m.optimizationStats
+}
+
 // ValuesContainer lets ValuesExpr and LiteralValuesExpr share code.
 type ValuesContainer interface {
 	RelExpr

pkg/sql/opt/memo/statistics_builder.go

@@ -4637,6 +4637,9 @@ func (sb *statisticsBuilder) clampSelForHistogram(
 		resClamp = props.MinSelectivity(resClamp,
 			props.MakeSelectivityFromFraction(oldHist.Resolution(), s.RowCount),
 		)
+		if resClamp.AsFloat() > clampedSel.AsFloat() {
+			sb.mem.optimizationStats.ClampedHistogramSelectivity = true
+		}
 		clampedSel = props.MaxSelectivity(clampedSel, resClamp)
 	}
 
@@ -4647,6 +4650,9 @@ func (sb *statisticsBuilder) clampSelForHistogram(
 		// scan at least 1/10000th of the table. This accounts for the possibility
 		// that the histogram missed extreme values due to sampling or staleness.
 		inequalityClamp := props.MakeSelectivity(histogramUnboundedInequalityMinSelectivity)
+		if inequalityClamp.AsFloat() > clampedSel.AsFloat() {
+			sb.mem.optimizationStats.ClampedInequalitySelectivity = true
+		}
 		clampedSel = props.MaxSelectivity(clampedSel, inequalityClamp)
 	}
 	return clampedSel

pkg/sql/plan_opt.go

@@ -263,6 +263,18 @@ func (p *planner) makeOptimizerPlan(ctx context.Context) error {
 		return err
 	}
 
+	// Log and increment telemetry counters for interesting decisions made during
+	// optimization.
+	optStats := execMemo.GetOptimizationStats()
+	if optStats.ClampedHistogramSelectivity {
+		log.VEventf(ctx, 2, "clamped histogram selectivity during optimization")
+		telemetry.Inc(sqltelemetry.PlanClampedHistogramSelectivityCounter)
+	}
+	if optStats.ClampedInequalitySelectivity {
+		log.VEventf(ctx, 2, "clamped inequality selectivity during optimization")
+		telemetry.Inc(sqltelemetry.PlanClampedInequalitySelectivityCounter)
+	}
+
 	// Build the plan tree.
 	const disableTelemetryAndPlanGists = false
 	return p.runExecBuild(ctx, execMemo, disableTelemetryAndPlanGists)

pkg/sql/sqltelemetry/planning.go

@@ -220,6 +220,16 @@ var PlanTypeAutoCustomCounter = telemetry.GetCounterOnce("sql.plan.type.auto-cus
 // used when plan_cache_mode=auto.
 var PlanTypeAutoGenericCounter = telemetry.GetCounterOnce("sql.plan.type.auto-generic")
 
+// PlanClampedHistogramSelectivityCounter is to be incremented whenever a plan
+// is used that clamped the selectivity estimate derived from a histogram to a
+// minimum value.
+var PlanClampedHistogramSelectivityCounter = telemetry.GetCounterOnce("sql.plan.clamped-histogram-selectivity")
+
+// PlanClampedInequalitySelectivityCounter is to be incremented whenever a plan
+// is used that clamped the selectivity estimate derived from an inequality over
+// a histogram to a minimum value.
+var PlanClampedInequalitySelectivityCounter = telemetry.GetCounterOnce("sql.plan.clamped-inequality-selectivity")
+
 // We can't parameterize these telemetry counters, so just make a bunch of
 // buckets for setting the join reorder limit since the range of reasonable
 // values for the join reorder limit is quite small.

pkg/sql/testdata/telemetry/planning

@@ -321,3 +321,119 @@ sql.plan.opt.order-by-nulls-non-standard
 feature-usage
 SELECT a FROM x ORDER BY b DESC NULLS LAST
 ----
+
+# Test for clamped histogram selectivity.
+
+feature-list
+sql.plan.clamped-histogram-selectivity
+sql.plan.clamped-inequality-selectivity
+----
+
+exec
+SET optimizer_clamp_low_histogram_selectivity = true;
+----
+
+exec
+SET optimizer_clamp_inequality_selectivity = true;
+----
+
+exec
+CREATE TYPE status_enum AS ENUM ('ACKNOWLEDGED', 'DROPPED', 'PENDING', 'SUCCEEDED');
+----
+
+exec
+CREATE TABLE t_large (
+  k INT PRIMARY KEY,
+  i INT,
+  s STRING,
+  e status_enum,
+  INDEX (i),
+  INDEX (s),
+  INDEX (i) WHERE e = 'PENDING'
+)
+----
+
+exec
+ALTER TABLE t_large INJECT STATISTICS '[
+  {
+    "columns": ["k"],
+    "created_at": "2018-01-01 1:00:00.00000+00:00",
+    "row_count": 1000000000,
+    "distinct_count": 1000000000,
+    "null_count": 0,
+    "avg_size": 2,
+    "histo_col_type": "int",
+    "histo_buckets": [
+      {"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
+      {"num_eq": 1, "num_range": 99000000, "distinct_range": 99000000, "upper_bound": "100000000"},
+      {"num_eq": 1, "num_range": 199000000, "distinct_range": 199000000, "upper_bound": "300000000"},
+      {"num_eq": 1, "num_range": 299000000, "distinct_range": 299000000, "upper_bound": "600000000"},
+      {"num_eq": 1, "num_range": 399000000, "distinct_range": 399000000, "upper_bound": "1000000000"}
+    ]
+  },
+  {
+    "columns": ["i"],
+    "created_at": "2018-01-01 1:00:00.00000+00:00",
+    "row_count": 1000000000,
+    "distinct_count": 44000000,
+    "null_count": 0,
+    "avg_size": 2,
+    "histo_col_type": "int",
+    "histo_buckets": [
+      {"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "0"},
+      {"num_eq": 10000000, "num_range": 90000000, "distinct_range": 10000000, "upper_bound": "100000000"},
+      {"num_eq": 10000000, "num_range": 190000000, "distinct_range": 10000000, "upper_bound": "200000000"},
+      {"num_eq": 20000000, "num_range": 280000000, "distinct_range": 10000000, "upper_bound": "300000000"},
+      {"num_eq": 30000000, "num_range": 370000000, "distinct_range": 10000000, "upper_bound": "400000000"}
+    ]
+  },
+  {
+    "columns": ["s"],
+    "created_at": "2018-01-01 1:00:00.00000+00:00",
+    "row_count": 1000000000,
+    "distinct_count": 400000000,
+    "null_count": 0,
+    "avg_size": 3,
+    "histo_col_type": "string",
+    "histo_buckets": [
+      {"num_eq": 0, "num_range": 0, "distinct_range": 0, "upper_bound": "apple"},
+      {"num_eq": 100000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "banana"},
+      {"num_eq": 100000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "cherry"},
+      {"num_eq": 200000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "mango"},
+      {"num_eq": 200000000, "num_range": 100000000, "distinct_range": 10000000, "upper_bound": "pineapple"}
+    ]
+  },
+  {
+    "columns": ["e"],
+    "created_at": "2018-01-01 1:00:00.00000+00:00",
+    "row_count": 1000000000,
+    "distinct_count": 4,
+    "null_count": 0,
+    "avg_size": 1,
+    "histo_col_type": "status_enum",
+    "histo_buckets": [
+      {"num_eq": 1000000, "num_range": 0, "distinct_range": 0, "upper_bound": "ACKNOWLEDGED"},
+      {"num_eq": 99000000, "num_range": 0, "distinct_range": 0, "upper_bound": "DROPPED"},
+      {"num_eq": 900000000, "num_range": 0, "distinct_range": 0, "upper_bound": "SUCCEEDED"}
+    ]
+  }
+]'
+----
+
+feature-usage
+SELECT * FROM t_large WHERE e = 'PENDING' AND k IN (110000000, 120000000, 130000000, 140000000)
+----
+sql.plan.clamped-histogram-selectivity
+
+feature-usage
+SELECT * FROM t_large WHERE k IN (100000000, 110000000, 120000000, 130000000) AND i > 500000000
+----
+sql.plan.clamped-histogram-selectivity
+sql.plan.clamped-inequality-selectivity
+
+# Cases where the selectivity clamps do not apply.
+feature-usage
+SELECT * FROM t_large WHERE e = 'SUCCEEDED';
+SELECT * FROM t_large;
+SELECT 1;
+----