Merge pull request #154755 from cockroachdb/blathers/backport-release-25.4-153419

release-25.4: sql/stats: merge all arbitrary partial and full table statistics

Author: Uzair5162

pkg/BUILD.bazel

@@ -2351,6 +2351,7 @@ GO_TARGETS = [
     "//pkg/sql/sem/plpgsqltree:plpgsqltree",
     "//pkg/sql/sem/semenumpb:semenumpb",
     "//pkg/sql/sem/transform:transform",
+    "//pkg/sql/sem/tree/datumrange:datumrange",
     "//pkg/sql/sem/tree/evalgen:evalgen",
     "//pkg/sql/sem/tree/evalgen:evalgen_lib",
     "//pkg/sql/sem/tree/treebin:treebin",

pkg/sql/distsql_plan_stats.go

@@ -304,6 +304,10 @@ func (dsp *DistSQLPlanner) createPartialStatsPlan(
 		return nil, pgerror.Newf(pgcode.FeatureNotSupported, "multi-column partial statistics are not currently supported")
 	}
 
+	if !reqStat.histogram {
+		return nil, pgerror.Newf(pgcode.FeatureNotSupported, "partial statistics without histograms are not supported")
+	}
+
 	var typeResolver *descs.DistSQLTypeResolver
 	if p := planCtx.planner; p != nil {
 		r := descs.NewDistSQLTypeResolver(p.Descriptors(), p.Txn())
@@ -378,6 +382,13 @@ func (dsp *DistSQLPlanner) createPartialStatsPlan(
 			"column %s does not have a prior statistic",
 			column.GetName())
 	}
+	if len(stat.Histogram) == 1 && stat.Histogram[0].UpperBound == tree.DNull {
+		return nil, pgerror.Newf(
+			pgcode.ObjectNotInPrerequisiteState,
+			"the latest full statistic histogram for column %s has only NULL values",
+			column.GetName(),
+		)
+	}
 
 	var predicate string
 	var prevLowerBound tree.Datum

pkg/sql/logictest/testdata/logic_test/constrained_stats

@@ -64,31 +64,6 @@ upper_bound  range_rows  distinct_range_rows  equal_rows
 50.00        0           0                    1
 55.00        0           0                    1
 
-statement ok
-CREATE STATISTICS stat_price_ranges ON price FROM products WHERE price <= 20.00 OR price > 40.00
-
-query TTIIIT colnames
-SELECT statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
-FROM [SHOW STATISTICS FOR TABLE products]
-WHERE statistics_name = 'stat_price_ranges'
-----
-statistics_name    column_names  row_count  distinct_count  null_count  partial_predicate
-stat_price_ranges  {price}       6          6               0           (price <= 20.00) OR (price > 40.00)
-
-let $hist_stat_price_ranges
-SELECT histogram_id FROM [SHOW STATISTICS FOR TABLE products] WHERE statistics_name = 'stat_price_ranges';
-
-query TIRI colnames,nosort
-SHOW HISTOGRAM $hist_stat_price_ranges
-----
-upper_bound  range_rows  distinct_range_rows  equal_rows
-10.00        0           0                    1
-15.00        0           0                    1
-20.00        0           0                    1
-45.00        0           0                    1
-50.00        0           0                    1
-55.00        0           0                    1
-
 statement ok
 CREATE STATISTICS stat_active_products ON status FROM products WHERE status = 'active'
 
@@ -111,17 +86,6 @@ WHERE statistics_name = 'stat_mid_range_products'
 statistics_name          column_names   row_count  distinct_count  null_count  partial_predicate
 stat_mid_range_products  {category_id}  8          2               0           category_id BETWEEN 1 AND 2
 
-statement ok
-CREATE STATISTICS stat_specific_categories ON category_id FROM products WHERE category_id IN (1, 3)
-
-query TTIIIT colnames
-SELECT statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
-FROM [SHOW STATISTICS FOR TABLE products]
-WHERE statistics_name = 'stat_specific_categories'
-----
-statistics_name           column_names   row_count  distinct_count  null_count  partial_predicate
-stat_specific_categories  {category_id}  6          2               0           category_id IN (1, 3)
-
 # Verify error cases
 statement error pq: partial statistics with WHERE must be on a single column
 CREATE STATISTICS stat_multi_col ON category_id, price FROM products WHERE status = 'active'
@@ -147,6 +111,12 @@ CREATE STATISTICS stat_subquery ON category_id FROM products WHERE category_id I
 statement error pq: predicate could not become a constrained scan of an index
 CREATE STATISTICS stat_func ON status FROM products WHERE length(status) > 6
 
+statement error pq: unimplemented: CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported
+CREATE STATISTICS stat_func ON price FROM products WHERE price < 20.00 OR price > 30.00
+
+statement error pq: unimplemented: CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported
+CREATE STATISTICS stat_specific_categories ON category_id FROM products WHERE category_id IN (1, 3)
+
 statement error syntax error
 CREATE STATISTICS stat_error2 ON price FROM products WHERE price >
 

pkg/sql/logictest/testdata/logic_test/distsql_stats

@@ -2675,6 +2675,18 @@ RESET enable_create_stats_using_extremes
 statement error pq: multi-column partial statistics are not currently supported
 CREATE STATISTICS abcd_a_b ON a, c FROM abcd USING EXTREMES;
 
+statement ok
+SET CLUSTER SETTING sql.stats.histogram_collection.enabled = false
+
+statement error pq: partial statistics without histograms are not supported
+CREATE STATISTICS abcd_a_b ON a FROM abcd USING EXTREMES;
+
+statement error pq: partial statistics without histograms are not supported
+CREATE STATISTICS abcd_a_b ON a FROM abcd WHERE a > 5;
+
+statement ok
+RESET CLUSTER SETTING sql.stats.histogram_collection.enabled
+
 # Verify that a non-inverted index string column with a string histogram
 # can have partial statistics
 statement ok
@@ -2730,9 +2742,15 @@ CREATE TABLE xyz (x INT, y INT, z INT, INDEX (x, y));
 statement error pq: column x does not have a prior statistic
 EXPLAIN ANALYZE CREATE STATISTICS xyz_x ON x FROM xyz USING EXTREMES;
 
+statement error pq: column x does not have a prior statistic
+EXPLAIN ANALYZE CREATE STATISTICS xyz_x ON x FROM xyz WHERE x > 5;
+
 statement error pq: the latest full statistic for column a has no histogram
 EXPLAIN ANALYZE CREATE STATISTICS u_partial ON a FROM u USING EXTREMES;
 
+statement error pq: the latest full statistic for column a has no histogram
+EXPLAIN ANALYZE CREATE STATISTICS u_partial ON a FROM u WHERE a > 5;
+
 statement error pq: table xy does not contain a non-partial forward index with y as a prefix column
 CREATE STATISTICS xy_y_partial ON y FROM xy USING EXTREMES;
 
@@ -2750,9 +2768,12 @@ CREATE STATISTICS only_null_stat ON a FROM only_null;
 statement ok
 SELECT crdb_internal.clear_table_stats_cache();
 
-statement error pq: only outer or NULL bounded buckets exist in only_null@only_null_a_idx \(table ID \d+, column IDs \[1\]\), so partial stats cannot be collected
+statement error pq: the latest full statistic histogram for column a has only NULL values
 EXPLAIN ANALYZE CREATE STATISTICS only_null_partial ON a FROM only_null USING EXTREMES;
 
+statement error pq: the latest full statistic histogram for column a has only NULL values
+EXPLAIN ANALYZE CREATE STATISTICS only_null_partial ON a FROM only_null WHERE a > 5;
+
 statement ok
 CREATE INDEX ON xy (y) WHERE y > 5;
 

pkg/sql/opt/cat/table.go

@@ -271,10 +271,8 @@ type TableStatistic interface {
 	// inverted index histograms, this will always return types.Bytes.
 	HistogramType() *types.T
 
-	// IsPartial returns true if this statistic was collected with a where
-	// clause. (If the where clause was something like "WHERE 1 = 1" or "WHERE
-	// true" this could technically be a full statistic rather than a partial
-	// statistic, but this function does not check for this.)
+	// IsPartial returns true if this statistic was collected with USING EXTREMES
+	// or with a WHERE clause.
 	IsPartial() bool
 
 	// IsMerged returns true if this statistic was created by merging a partial

pkg/sql/opt/exec/execbuilder/testdata/partial_stats

@@ -1478,3 +1478,125 @@ vectorized: true
   spans: [/10 - /10]
 
 subtest end
+
+# Ensure that constrained partial statistics are merged into full statistics
+# and used by the optimizer.
+
+statement ok
+CREATE TABLE m (b INT PRIMARY KEY)
+
+statement ok
+ALTER TABLE m INJECT STATISTICS '[
+  {
+    "id": 1,
+    "avg_size": 1,
+    "columns": ["b"],
+    "created_at": "2000-01-01 00:00:00.000000",
+    "distinct_count": 4,
+    "histo_buckets": [
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "10"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "20"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "30"},
+      {"distinct_range": 0, "num_eq": 1, "num_range": 0, "upper_bound": "40"}
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 2,
+    "name": "full",
+    "null_count": 0,
+    "row_count": 4
+  },
+  {
+    "id": 2,
+    "avg_size": 1,
+    "columns": ["b"],
+    "created_at": "2000-01-02 00:00:00.000000",
+    "distinct_count": 4,
+    "histo_buckets": [
+      {"distinct_range": 0, "num_eq": 3, "num_range": 0, "upper_bound": "20"},
+      {"distinct_range": 2, "num_eq": 2, "num_range": 3, "upper_bound": "30"}
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 2,
+    "name": "partial",
+    "null_count": 0,
+    "partial_predicate": "((b > 15:::INT8) AND (b < 35:::INT8))",
+    "row_count": 8,
+    "full_statistic_id": 1
+  }
+]'
+
+query TTTIIII colnames
+SELECT statistics_name, column_names, created, row_count, distinct_count, null_count, avg_size
+FROM [SHOW STATISTICS FOR TABLE m WITH MERGE]
+ORDER BY created
+----
+statistics_name  column_names  created                        row_count  distinct_count  null_count  avg_size
+full             {b}           2000-01-01 00:00:00 +0000 UTC  4          4               0           1
+partial          {b}           2000-01-02 00:00:00 +0000 UTC  8          4               0           1
+__merged__       {b}           2000-01-02 00:00:00 +0000 UTC  10         6               0           1
+
+query T
+SELECT jsonb_pretty(stat)
+FROM (
+SELECT jsonb_array_elements(statistics) AS stat FROM [SHOW STATISTICS USING JSON FOR TABLE m WITH MERGE]
+)
+WHERE stat->>'name' = '__merged__';
+----
+{
+    "avg_size": 1,
+    "columns": [
+        "b"
+    ],
+    "created_at": "2000-01-02 00:00:00",
+    "distinct_count": 6,
+    "histo_buckets": [
+        {
+            "distinct_range": 0,
+            "num_eq": 1,
+            "num_range": 0,
+            "upper_bound": "10"
+        },
+        {
+            "distinct_range": 0,
+            "num_eq": 3,
+            "num_range": 0,
+            "upper_bound": "20"
+        },
+        {
+            "distinct_range": 2,
+            "num_eq": 2,
+            "num_range": 3,
+            "upper_bound": "30"
+        },
+        {
+            "distinct_range": 0,
+            "num_eq": 1,
+            "num_range": 0,
+            "upper_bound": "40"
+        }
+    ],
+    "histo_col_type": "INT8",
+    "histo_version": 3,
+    "name": "__merged__",
+    "null_count": 0,
+    "row_count": 10
+}
+
+# Now clear the stats cache so that the query below is guaranteed to pick up the
+# new stats (partial and merged).
+statement ok
+SELECT crdb_internal.clear_table_stats_cache();
+
+query T
+EXPLAIN (OPT, VERBOSE) SELECT * FROM m WHERE b < 35
+----
+scan m
+ ├── columns: b:1
+ ├── constraint: /1: [ - /34]
+ ├── stats: [rows=9, distinct(1)=5, null(1)=0]
+ │   histogram(1)=  0  1   0  3   3  2   0  0
+ │                <--- 10 --- 20 --- 30 --- 34
+ ├── cost: 27.11
+ ├── cost-flags: unbounded-cardinality
+ ├── key: (1)
+ └── distribution: test

pkg/sql/opt/optbuilder/misc_statements.go

@@ -323,5 +323,10 @@ func (b *Builder) buildWhereForStatistics(
 			"predicate is a contradiction"))
 	}
 
+	if cons.Spans.Count() > 1 {
+		panic(unimplemented.NewWithIssue(154040,
+			"CREATE STATISTICS with a WHERE clause that produces multiple spans is not yet supported"))
+	}
+
 	return indexOrd, &cons
 }

pkg/sql/opt/props/BUILD.bazel

@@ -23,14 +23,12 @@ go_library(
         "//pkg/sql/opt",
         "//pkg/sql/opt/cat",
         "//pkg/sql/opt/constraint",
-        "//pkg/sql/rowenc/keyside",
         "//pkg/sql/sem/eval",
         "//pkg/sql/sem/tree",
+        "//pkg/sql/sem/tree/datumrange",
         "//pkg/sql/sem/volatility",
         "//pkg/sql/types",
         "//pkg/util/buildutil",
-        "//pkg/util/encoding",
-        "//pkg/util/timetz",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_olekukonko_tablewriter//:tablewriter",