sql/stats: merge all arbitrary partial and full table statistics

This commit generalizes partial and full stat merging in the stats cache
to support partial stats collected over any range of values in a column.
Non-extreme partial stats are treated as covering a single span, and
extreme partials continue to separate the lower and upper extreme
buckets as before.

Additionally, merged stats now apply all partial stats collected after
the latest full stat, in order of creation. Previously, we could only
merge partial stats collected at the extremes of the index, and would
only merge the latest one.

Part of: #93998

Release note (sql change): The optimizer can now use table statistics
that merge the latest full statistic with all newer partial statistics,
including those over arbitrary constraints over a single span.

Author: Uzair5162

pkg/BUILD.bazel

@@ -2338,6 +2338,7 @@ GO_TARGETS = [
     "//pkg/sql/sem/plpgsqltree:plpgsqltree",
     "//pkg/sql/sem/semenumpb:semenumpb",
     "//pkg/sql/sem/transform:transform",
+    "//pkg/sql/sem/tree/datumrange:datumrange",
     "//pkg/sql/sem/tree/evalgen:evalgen",
     "//pkg/sql/sem/tree/evalgen:evalgen_lib",
     "//pkg/sql/sem/tree/treebin:treebin",

pkg/sql/distsql_plan_stats.go

@@ -304,6 +304,10 @@ func (dsp *DistSQLPlanner) createPartialStatsPlan(
 		return nil, pgerror.Newf(pgcode.FeatureNotSupported, "multi-column partial statistics are not currently supported")
 	}
 
+	if !reqStat.histogram {
+		return nil, pgerror.Newf(pgcode.FeatureNotSupported, "partial statistics without histograms are not supported")
+	}
+
 	var typeResolver *descs.DistSQLTypeResolver
 	if p := planCtx.planner; p != nil {
 		r := descs.NewDistSQLTypeResolver(p.Descriptors(), p.Txn())
@@ -378,6 +382,13 @@ func (dsp *DistSQLPlanner) createPartialStatsPlan(
 			"column %s does not have a prior statistic",
 			column.GetName())
 	}
+	if len(stat.Histogram) == 1 && stat.Histogram[0].UpperBound == tree.DNull {
+		return nil, pgerror.Newf(
+			pgcode.ObjectNotInPrerequisiteState,
+			"the latest full statistic histogram for column %s has only NULL values",
+			column.GetName(),
+		)
+	}
 
 	var predicate string
 	var prevLowerBound tree.Datum

pkg/sql/logictest/testdata/logic_test/constrained_stats

@@ -64,31 +64,6 @@ upper_bound  range_rows  distinct_range_rows  equal_rows
 50.00        0           0                    1
 55.00        0           0                    1
 
-statement ok
-CREATE STATISTICS stat_price_ranges ON price FROM products WHERE price <= 20.00 OR price > 40.00
-
-query TTIIIT colnames
-SELECT statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
-FROM [SHOW STATISTICS FOR TABLE products]
-WHERE statistics_name = 'stat_price_ranges'
-----
-statistics_name    column_names  row_count  distinct_count  null_count  partial_predicate
-stat_price_ranges  {price}       6          6               0           (price <= 20.00) OR (price > 40.00)
-
-let $hist_stat_price_ranges
-SELECT histogram_id FROM [SHOW STATISTICS FOR TABLE products] WHERE statistics_name = 'stat_price_ranges';
-
-query TIRI colnames,nosort
-SHOW HISTOGRAM $hist_stat_price_ranges
-----
-upper_bound  range_rows  distinct_range_rows  equal_rows
-10.00        0           0                    1
-15.00        0           0                    1
-20.00        0           0                    1
-45.00        0           0                    1
-50.00        0           0                    1
-55.00        0           0                    1
-
 statement ok
 CREATE STATISTICS stat_active_products ON status FROM products WHERE status = 'active'
 
@@ -111,17 +86,6 @@ WHERE statistics_name = 'stat_mid_range_products'
 statistics_name          column_names   row_count  distinct_count  null_count  partial_predicate
 stat_mid_range_products  {category_id}  8          2               0           category_id BETWEEN 1 AND 2
 
-statement ok
-CREATE STATISTICS stat_specific_categories ON category_id FROM products WHERE category_id IN (1, 3)
-
-query TTIIIT colnames
-SELECT statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
-FROM [SHOW STATISTICS FOR TABLE products]
-WHERE statistics_name = 'stat_specific_categories'
-----
-statistics_name           column_names   row_count  distinct_count  null_count  partial_predicate
-stat_specific_categories  {category_id}  6          2               0           category_id IN (1, 3)
-
 # Verify error cases
 statement error pq: partial statistics with WHERE must be on a single column
 CREATE STATISTICS stat_multi_col ON category_id, price FROM products WHERE status = 'active'
@@ -147,6 +111,12 @@ CREATE STATISTICS stat_subquery ON category_id FROM products WHERE category_id I
 statement error pq: predicate could not become a constrained scan of an index
 CREATE STATISTICS stat_func ON status FROM products WHERE length(status) > 6
 
+statement error pq: unimplemented: CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported
+CREATE STATISTICS stat_func ON price FROM products WHERE price < 20.00 OR price > 30.00
+
+statement error pq: unimplemented: CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported
+CREATE STATISTICS stat_specific_categories ON category_id FROM products WHERE category_id IN (1, 3)
+
 statement error syntax error
 CREATE STATISTICS stat_error2 ON price FROM products WHERE price >
 

pkg/sql/logictest/testdata/logic_test/distsql_stats

@@ -2675,6 +2675,18 @@ RESET enable_create_stats_using_extremes
 statement error pq: multi-column partial statistics are not currently supported
 CREATE STATISTICS abcd_a_b ON a, c FROM abcd USING EXTREMES;
 
+statement ok
+SET CLUSTER SETTING sql.stats.histogram_collection.enabled = false
+
+statement error pq: partial statistics without histograms are not supported
+CREATE STATISTICS abcd_a_b ON a FROM abcd USING EXTREMES;
+
+statement error pq: partial statistics without histograms are not supported
+CREATE STATISTICS abcd_a_b ON a FROM abcd WHERE a > 5;
+
+statement ok
+RESET CLUSTER SETTING sql.stats.histogram_collection.enabled
+
 # Verify that a non-inverted index string column with a string histogram
 # can have partial statistics
 statement ok
@@ -2730,9 +2742,15 @@ CREATE TABLE xyz (x INT, y INT, z INT, INDEX (x, y));
 statement error pq: column x does not have a prior statistic
 EXPLAIN ANALYZE CREATE STATISTICS xyz_x ON x FROM xyz USING EXTREMES;
 
+statement error pq: column x does not have a prior statistic
+EXPLAIN ANALYZE CREATE STATISTICS xyz_x ON x FROM xyz WHERE x > 5;
+
 statement error pq: the latest full statistic for column a has no histogram
 EXPLAIN ANALYZE CREATE STATISTICS u_partial ON a FROM u USING EXTREMES;
 
+statement error pq: the latest full statistic for column a has no histogram
+EXPLAIN ANALYZE CREATE STATISTICS u_partial ON a FROM u WHERE a > 5;
+
 statement error pq: table xy does not contain a non-partial forward index with y as a prefix column
 CREATE STATISTICS xy_y_partial ON y FROM xy USING EXTREMES;
 
@@ -2750,9 +2768,12 @@ CREATE STATISTICS only_null_stat ON a FROM only_null;
 statement ok
 SELECT crdb_internal.clear_table_stats_cache();
 
-statement error pq: only outer or NULL bounded buckets exist in only_null@only_null_a_idx \(table ID \d+, column IDs \[1\]\), so partial stats cannot be collected
+statement error pq: the latest full statistic histogram for column a has only NULL values
 EXPLAIN ANALYZE CREATE STATISTICS only_null_partial ON a FROM only_null USING EXTREMES;
 
+statement error pq: the latest full statistic histogram for column a has only NULL values
+EXPLAIN ANALYZE CREATE STATISTICS only_null_partial ON a FROM only_null WHERE a > 5;
+
 statement ok
 CREATE INDEX ON xy (y) WHERE y > 5;
 

pkg/sql/opt/cat/table.go

@@ -271,10 +271,8 @@ type TableStatistic interface {
 	// inverted index histograms, this will always return types.Bytes.
 	HistogramType() *types.T
 
-	// IsPartial returns true if this statistic was collected with a where
-	// clause. (If the where clause was something like "WHERE 1 = 1" or "WHERE
-	// true" this could technically be a full statistic rather than a partial
-	// statistic, but this function does not check for this.)
+	// IsPartial returns true if this statistic was collected with USING EXTREMES
+	// or with a WHERE clause.
 	IsPartial() bool
 
 	// IsMerged returns true if this statistic was created by merging a partial

pkg/sql/opt/exec/execbuilder/testdata/partial_stats

@@ -1478,3 +1478,125 @@ vectorized: true
   spans: [/10 - /10]
 
 subtest end
+
+# Ensure that constrained partial statistics are merged into full statistics
+# and used by the optimizer.
+
+statement ok
+CREATE TABLE m (b INT PRIMARY KEY)
+
+statement ok
+ALTER TABLE m INJECT STATISTICS '[
+  {
+    "id": 1,
+    "avg_size": 1,
+    "columns": ["b"],
+    "created_at": "2000-01-01 00:00:00.000000",
+    "distinct_count": 4,
+    "histo_buckets": [
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "10"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "20"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "30"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "40"}
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 2,
+    "name": "full",
+    "null_count": 0,
+    "row_count": 4
+  },
+  {
+    "id": 2,
+    "avg_size": 1,
+    "columns": ["b"],
+    "created_at": "2000-01-02 00:00:00.000000",
+    "distinct_count": 4,
+    "histo_buckets": [
+      {"distinct_range": 0, "num_eq": 3, "num_range": 0, "upper_bound": "20"},
+      {"distinct_range": 2, "num_eq": 2, "num_range": 3, "upper_bound": "30"}
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 2,
+    "name": "partial",
+    "null_count": 0,
+    "partial_predicate": "((b > 15:::INT8) AND (b < 35:::INT8))",
+    "row_count": 8,
+    "full_statistic_id": 1
+  }
+]'
+
+query TTTIIII colnames
+SELECT statistics_name, column_names, created, row_count, distinct_count, null_count, avg_size
+FROM [SHOW STATISTICS FOR TABLE m WITH MERGE]
+ORDER BY created
+----
+statistics_name  column_names  created                        row_count  distinct_count  null_count  avg_size
+full             {b}           2000-01-01 00:00:00 +0000 UTC  4          4               0           1
+partial          {b}           2000-01-02 00:00:00 +0000 UTC  8          4               0           1
+__merged__       {b}           2000-01-02 00:00:00 +0000 UTC  10         6               0           1
+
+query T
+SELECT jsonb_pretty(stat)
+FROM (
+SELECT jsonb_array_elements(statistics) AS stat FROM [SHOW STATISTICS USING JSON FOR TABLE m WITH MERGE]
+)
+WHERE stat->>'name' = '__merged__';
+----
+{
+    "avg_size": 1,
+    "columns": [
+        "b"
+    ],
+    "created_at": "2000-01-02 00:00:00",
+    "distinct_count": 6,
+    "histo_buckets": [
+        {
+            "distinct_range": 0,
+            "num_eq": 1,
+            "num_range": 0,
+            "upper_bound": "10"
+        },
+        {
+            "distinct_range": 0,
+            "num_eq": 3,
+            "num_range": 0,
+            "upper_bound": "20"
+        },
+        {
+            "distinct_range": 2,
+            "num_eq": 2,
+            "num_range": 3,
+            "upper_bound": "30"
+        },
+        {
+            "distinct_range": 0,
+            "num_eq": 1,
+            "num_range": 0,
+            "upper_bound": "40"
+        }
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 3,
+    "name": "__merged__",
+    "null_count": 0,
+    "row_count": 10
+}
+
+# Now clear the stats cache so that the query below is guaranteed to pick up the
+# new stats (partial and merged).
+statement ok
+SELECT crdb_internal.clear_table_stats_cache();
+
+query T
+EXPLAIN (OPT, VERBOSE) SELECT * FROM m WHERE b < 35
+----
+scan m
+ ├── columns: b:1
+ ├── constraint: /1: [ - /34]
+ ├── stats: [rows=9, distinct(1)=5, null(1)=0]
+ │   histogram(1)=  0  1   0  3   3  2   0  0
+ │                <--- 10 --- 20 --- 30 --- 34
+ ├── cost: 27.11
+ ├── cost-flags: unbounded-cardinality
+ ├── key: (1)
+ └── distribution: test

pkg/sql/opt/optbuilder/misc_statements.go

@@ -323,5 +323,10 @@ func (b *Builder) buildWhereForStatistics(
 			"predicate is a contradiction"))
 	}
 
+	if cons.Spans.Count() > 1 {
+		panic(unimplemented.NewWithIssue(154040,
+			"CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported"))
+	}
+
 	return indexOrd, &cons
 }

pkg/sql/opt/props/BUILD.bazel

@@ -23,14 +23,12 @@ go_library(
         "//pkg/sql/opt",
         "//pkg/sql/opt/cat",
         "//pkg/sql/opt/constraint",
-        "//pkg/sql/rowenc/keyside",
         "//pkg/sql/sem/eval",
         "//pkg/sql/sem/tree",
+        "//pkg/sql/sem/tree/datumrange",
         "//pkg/sql/sem/volatility",
         "//pkg/sql/types",
         "//pkg/util/buildutil",
-        "//pkg/util/encoding",
-        "//pkg/util/timetz",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_olekukonko_tablewriter//:tablewriter",