Merge #140531 #142924

140531: gceworker: better handling of firewall for home use r=dhartunian a=dhartunian

Previously, creating a gceworker from outside the office would fail because firewall rules wouldn't get updated prior to ssh-ing in.

We now update the firewall prior to ssh-ing, and also delete the home ssh rule on destroy.

Release note: None

142924: sql: support disabling full auto stats independent of partial auto stats r=rytaft a=rytaft

This commit adds support for enabling and disabling full auto stats collection independent of whether or not partial auto stats are enabled/disabled.

This commit also fixes a flake in the `distsql_automatic_partial_stats` logictest.

Fixes #132697

Release note (sql change): It is now possible to automatically collect partial table statistics, but disable automatic collection of full table statistics. The new cluster and table settings to enable/ disable automatic collection of full table statistics (without affecting the settings for partial stats) are:
- `sql.stats.automatic_full_collection.enabled`/ `sql_stats_automatic_full_collection_enabled`
  - Defaults to `true`

Co-authored-by: David Hartunian <[email protected]>
Co-authored-by: Rebecca Taft <[email protected]>

Author: 3 people

docs/generated/settings/settings-for-tenants.txt

@@ -343,6 +343,7 @@ sql.stats.activity.persisted_rows.max	integer	200000	maximum number of rows of s
 sql.stats.automatic_collection.enabled	boolean	true	automatic statistics collection mode	application
 sql.stats.automatic_collection.fraction_stale_rows	float	0.2	target fraction of stale rows per table that will trigger a statistics refresh	application
 sql.stats.automatic_collection.min_stale_rows	integer	500	target minimum number of stale rows per table that will trigger a statistics refresh	application
+sql.stats.automatic_full_collection.enabled	boolean	true	automatic full statistics collection mode	application
 sql.stats.automatic_partial_collection.enabled	boolean	true	automatic partial statistics collection mode	application
 sql.stats.automatic_partial_collection.fraction_stale_rows	float	0.05	target fraction of stale rows per table that will trigger a partial statistics refresh	application
 sql.stats.automatic_partial_collection.min_stale_rows	integer	100	target minimum number of stale rows per table that will trigger a partial statistics refresh	application

docs/generated/settings/settings.html

@@ -298,6 +298,7 @@
 <tr><td><div id="setting-sql-stats-automatic-collection-enabled" class="anchored"><code>sql.stats.automatic_collection.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>automatic statistics collection mode</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-collection-fraction-stale-rows" class="anchored"><code>sql.stats.automatic_collection.fraction_stale_rows</code></div></td><td>float</td><td><code>0.2</code></td><td>target fraction of stale rows per table that will trigger a statistics refresh</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-collection-min-stale-rows" class="anchored"><code>sql.stats.automatic_collection.min_stale_rows</code></div></td><td>integer</td><td><code>500</code></td><td>target minimum number of stale rows per table that will trigger a statistics refresh</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-sql-stats-automatic-full-collection-enabled" class="anchored"><code>sql.stats.automatic_full_collection.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>automatic full statistics collection mode</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-enabled" class="anchored"><code>sql.stats.automatic_partial_collection.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>automatic partial statistics collection mode</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-fraction-stale-rows" class="anchored"><code>sql.stats.automatic_partial_collection.fraction_stale_rows</code></div></td><td>float</td><td><code>0.05</code></td><td>target fraction of stale rows per table that will trigger a partial statistics refresh</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-stats-automatic-partial-collection-min-stale-rows" class="anchored"><code>sql.stats.automatic_partial_collection.min_stale_rows</code></div></td><td>integer</td><td><code>100</code></td><td>target minimum number of stale rows per table that will trigger a partial statistics refresh</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/sql/catalog/catpb/catalog.go

@@ -9,10 +9,14 @@ package catpb
 // enabled table setting is enabled, disabled, or not set.
 type AutoStatsCollectionStatus int
 
-// AutoPartialStatsCollectionStatus represents whether the auto stats
+// AutoPartialStatsCollectionStatus represents whether the auto partial stats
 // collections enabled table setting is enabled, disabled or not set.
 type AutoPartialStatsCollectionStatus int
 
+// AutoFullStatsCollectionStatus represents whether the auto full stats
+// collections enabled table setting is enabled, disabled or not set.
+type AutoFullStatsCollectionStatus int
+
 // The values for AutoStatsCollectionStatus.
 const (
 	AutoStatsCollectionNotSet AutoStatsCollectionStatus = iota
@@ -26,6 +30,12 @@ const (
 	AutoPartialStatsCollectionDisabled
 )
 
+const (
+	AutoFullStatsCollectionNotSet AutoFullStatsCollectionStatus = iota
+	AutoFullStatsCollectionEnabled
+	AutoFullStatsCollectionDisabled
+)
+
 const (
 	// AutoStatsEnabledSettingName is the name of the automatic stats collection
 	// enabled cluster setting.
@@ -60,15 +70,15 @@ const (
 	AutoStatsFractionStaleTableSettingName = "sql_stats_automatic_collection_fraction_stale_rows"
 
 	// AutoPartialStatsEnabledSettingName is the name of the automatic partial
-	// stats collection enabled cluster setting
+	// stats collection enabled cluster setting.
 	AutoPartialStatsEnabledSettingName = "sql.stats.automatic_partial_collection.enabled"
 
 	// AutoPartialStatsEnabledTableSettingName is the name of the automatic
 	// partial stats collection enabled table setting.
 	AutoPartialStatsEnabledTableSettingName = "sql_stats_automatic_partial_collection_enabled"
 
 	// AutoPartialStatsMinStaleSettingName is the name of the automatic partial
-	// stats collection min stale rows cluster setting
+	// stats collection min stale rows cluster setting.
 	AutoPartialStatsMinStaleSettingName = "sql.stats.automatic_partial_collection.min_stale_rows"
 
 	// AutoPartialStatsMinStaleTableSettingName is the name of the automatic
@@ -82,6 +92,14 @@ const (
 	// AutoPartialStatsFractionStaleTableSettingName is the name of the automatic
 	// partial stats collection fraction stale rows table setting.
 	AutoPartialStatsFractionStaleTableSettingName = "sql_stats_automatic_partial_collection_fraction_stale_rows"
+
+	// AutoFullStatsEnabledSettingName is the name of the automatic full
+	// stats collection enabled cluster setting.
+	AutoFullStatsEnabledSettingName = "sql.stats.automatic_full_collection.enabled"
+
+	// AutoFullStatsEnabledTableSettingName is the name of the automatic
+	// full stats collection enabled table setting.
+	AutoFullStatsEnabledTableSettingName = "sql_stats_automatic_full_collection_enabled"
 )
 
 // AutoStatsCollectionEnabled indicates if automatic statistics collection is
@@ -124,6 +142,7 @@ func (as *AutoStatsSettings) NoAutoStatsSettingsOverrides() bool {
 		as.MinStaleRows != nil ||
 		as.FractionStaleRows != nil ||
 		as.PartialEnabled != nil ||
+		as.FullEnabled != nil ||
 		as.PartialMinStaleRows != nil ||
 		as.PartialFractionStaleRows != nil {
 		return false
@@ -151,6 +170,18 @@ func (as *AutoStatsSettings) AutoPartialStatsCollectionEnabled() AutoPartialStat
 	return AutoPartialStatsCollectionDisabled
 }
 
+// AutoFullStatsCollectionEnabled indicates if automatic full statistics
+// collection is explicitly enabled or disabled.
+func (as *AutoStatsSettings) AutoFullStatsCollectionEnabled() AutoFullStatsCollectionStatus {
+	if as.FullEnabled == nil {
+		return AutoFullStatsCollectionNotSet
+	}
+	if *as.FullEnabled {
+		return AutoFullStatsCollectionEnabled
+	}
+	return AutoFullStatsCollectionDisabled
+}
+
 // AutoPartialStatsMinStaleRows indicates the setting of
 // sql_stats_automatic_partial_collection_min_stale_rows in
 // AutoStatsSettings. If ok is true, then the minStaleRows value is

pkg/sql/catalog/catpb/catalog.proto

@@ -213,7 +213,9 @@ message AutoStatsSettings {
   // PartialEnabled is table setting sql_stats_automatic_partial_collection_enabled.
   optional bool partial_enabled = 4;
   // PartialMinStaleRows is table setting sql_stats_automatic_partial_collection_min_stale_rows.
-  optional int64  partial_min_stale_rows = 5;
+  optional int64 partial_min_stale_rows = 5;
   // PartialFractionStaleRows is table setting sql_stats_automatic_partial_collection_fraction_stale_rows.
   optional double partial_fraction_stale_rows = 6;
+  // FullEnabled is table setting sql_stats_automatic_full_collection_enabled.
+  optional bool full_enabled = 7;
 }

pkg/sql/catalog/descriptor.go

@@ -796,6 +796,9 @@ type TableDescriptor interface {
 	// AutoPartialStatsCollectionEnabled indicates if automatic partial statistics
 	// collection is explicitly enabled or disabled for this table.
 	AutoPartialStatsCollectionEnabled() catpb.AutoPartialStatsCollectionStatus
+	// AutoFullStatsCollectionEnabled indicates if automatic full statistics
+	// collection is explicitly enabled or disabled for this table.
+	AutoFullStatsCollectionEnabled() catpb.AutoFullStatsCollectionStatus
 	// AutoStatsMinStaleRows indicates the setting of
 	// sql_stats_automatic_collection_min_stale_rows for this table.
 	// If ok is true, then the minStaleRows value is valid, otherwise this has not

pkg/sql/catalog/tabledesc/structured.go

@@ -2519,6 +2519,11 @@ func (desc *wrapper) GetStorageParams(spaceBetweenEqual bool) []string {
 			appendStorageParam(catpb.AutoPartialStatsEnabledTableSettingName,
 				fmt.Sprintf("%v", value))
 		}
+		if settings.FullEnabled != nil {
+			value := *settings.FullEnabled
+			appendStorageParam(catpb.AutoFullStatsEnabledTableSettingName,
+				fmt.Sprintf("%v", value))
+		}
 		if settings.PartialMinStaleRows != nil {
 			value := *settings.PartialMinStaleRows
 			appendStorageParam(catpb.AutoPartialStatsMinStaleTableSettingName,
@@ -2576,13 +2581,22 @@ func (desc *wrapper) AutoStatsCollectionEnabled() catpb.AutoStatsCollectionStatu
 	return desc.AutoStatsSettings.AutoStatsCollectionEnabled()
 }
 
+// AutoPartialStatsCollectionEnabled implements the TableDescriptor interface.
 func (desc *wrapper) AutoPartialStatsCollectionEnabled() catpb.AutoPartialStatsCollectionStatus {
 	if desc.AutoStatsSettings == nil {
 		return catpb.AutoPartialStatsCollectionNotSet
 	}
 	return desc.AutoStatsSettings.AutoPartialStatsCollectionEnabled()
 }
 
+// AutoFullStatsCollectionEnabled implements the TableDescriptor interface.
+func (desc *wrapper) AutoFullStatsCollectionEnabled() catpb.AutoFullStatsCollectionStatus {
+	if desc.AutoStatsSettings == nil {
+		return catpb.AutoFullStatsCollectionNotSet
+	}
+	return desc.AutoStatsSettings.AutoFullStatsCollectionEnabled()
+}
+
 // AutoStatsMinStaleRows implements the TableDescriptor interface.
 func (desc *wrapper) AutoStatsMinStaleRows() (minStaleRows int64, ok bool) {
 	if desc.AutoStatsSettings == nil {

pkg/sql/catalog/tabledesc/validate.go

@@ -2238,6 +2238,7 @@ func (desc *wrapper) validateAutoStatsSettings(vea catalog.ValidationErrorAccumu
 	}
 	desc.validateAutoStatsEnabled(vea, catpb.AutoStatsEnabledTableSettingName, desc.AutoStatsSettings.Enabled)
 	desc.validateAutoStatsEnabled(vea, catpb.AutoPartialStatsEnabledTableSettingName, desc.AutoStatsSettings.PartialEnabled)
+	desc.validateAutoStatsEnabled(vea, catpb.AutoFullStatsEnabledTableSettingName, desc.AutoStatsSettings.FullEnabled)
 
 	desc.validateMinStaleRows(vea, catpb.AutoStatsMinStaleTableSettingName, desc.AutoStatsSettings.MinStaleRows)
 	desc.validateMinStaleRows(vea, catpb.AutoPartialStatsMinStaleTableSettingName, desc.AutoStatsSettings.PartialMinStaleRows)

pkg/sql/catalog/tabledesc/validate_test.go

@@ -313,6 +313,7 @@ var validationMap = []struct {
 			"MinStaleRows":             {status: iSolemnlySwearThisFieldIsValidated},
 			"FractionStaleRows":        {status: iSolemnlySwearThisFieldIsValidated},
 			"PartialEnabled":           {status: iSolemnlySwearThisFieldIsValidated},
+			"FullEnabled":              {status: iSolemnlySwearThisFieldIsValidated},
 			"PartialMinStaleRows":      {status: iSolemnlySwearThisFieldIsValidated},
 			"PartialFractionStaleRows": {status: iSolemnlySwearThisFieldIsValidated},
 		},
@@ -2566,6 +2567,18 @@ func TestValidateTableDesc(t *testing.T) {
 				NextColumnID:      2,
 				AutoStatsSettings: &catpb.AutoStatsSettings{PartialEnabled: &boolTrue},
 			}},
+		{err: `Setting sql_stats_automatic_full_collection_enabled may not be set on virtual table`,
+			desc: descpb.TableDescriptor{
+				ID:            catconstants.MinVirtualID,
+				ParentID:      1,
+				Name:          "foo",
+				FormatVersion: descpb.InterleavedFormatVersion,
+				Columns: []descpb.ColumnDescriptor{
+					{ID: 1, Name: "bar"},
+				},
+				NextColumnID:      2,
+				AutoStatsSettings: &catpb.AutoStatsSettings{FullEnabled: &boolTrue},
+			}},
 		{err: `Setting sql_stats_automatic_collection_enabled may not be set on a view or sequence`,
 			desc: descpb.TableDescriptor{
 				Name:                    "bar",

pkg/sql/logictest/testdata/logic_test/distsql_automatic_partial_stats

@@ -29,10 +29,21 @@ statistics_name   column_names  row_count  distinct_count  null_count  partial_p
 statement ok
 CREATE STATISTICS __auto__ FROM data
 
-# Set the min_stale_rows to MaxInt32 to ensure that full stat collections are
-# not triggered
+query TTIIIT colnames,retry
+SELECT DISTINCT ON (statistics_name, column_names) statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
+FROM [SHOW STATISTICS FOR TABLE data] ORDER BY statistics_name, column_names, created DESC
+----
+statistics_name  column_names  row_count  distinct_count  null_count  partial_predicate
+__auto__         {a}           1000       10              0           NULL
+__auto__         {a,b}         1000       100             0           NULL
+__auto__         {a,b,c}       1000       1000            0           NULL
+__auto__         {b}           1000       10              0           NULL
+__auto__         {c}           1000       10              0           NULL
+__auto__         {d}           1000       1               0           NULL
+
+# Disable full stat collections.
 statement ok
-SET CLUSTER SETTING sql.stats.automatic_collection.min_stale_rows = 2147483647
+SET CLUSTER SETTING sql.stats.automatic_full_collection.enabled = false
 
 statement ok
 SET CLUSTER SETTING sql.stats.automatic_collection.enabled = true
@@ -60,11 +71,16 @@ __auto_partial__  {a}           0          0               0           (a IS NUL
 __auto_partial__  {c}           0          0               0           (c IS NULL) OR ((c < 1.0:::FLOAT8) OR (c > 10.0:::FLOAT8))
 __auto_partial__  {d}           100        1               0           (d IS NULL) OR ((d < 1:::DECIMAL) OR (d > 1:::DECIMAL))
 
-# Disable automatic partial stats with the table setting
+# Enable full stat collections with the cluster setting.
 statement ok
-ALTER TABLE data SET (sql_stats_automatic_partial_collection_enabled = false)
+SET CLUSTER SETTING sql.stats.automatic_full_collection.enabled = true
 
-# Change 20% of the table, no new partial stats should be collected.
+# Disable automatic partial and full stats with the table settings.
+statement ok
+ALTER TABLE data SET (sql_stats_automatic_partial_collection_enabled = false);
+ALTER TABLE data SET (sql_stats_automatic_full_collection_enabled = false)
+
+# Change 20% of the table, no new partial or full stats should be collected.
 statement ok
 UPDATE DATA SET d = 3 WHERE a = 1 OR a = 2
 
@@ -175,3 +191,27 @@ __auto__          {d}           1000       1               0           NULL
 __auto_partial__  {a}           16         1               0           (a IS NULL) OR ((a < 1:::INT8) OR (a > 10:::INT8))
 __auto_partial__  {c}           16         4               0           (c IS NULL) OR ((c < 1.0:::FLOAT8) OR (c > 10.0:::FLOAT8))
 __auto_partial__  {d}           216        3               4           (d IS NULL) OR ((d < 1:::DECIMAL) OR (d > 1:::DECIMAL))
+
+# Now disable automatic partial stats but enable full stats with the table settings.
+statement ok
+ALTER TABLE data SET (sql_stats_automatic_partial_collection_enabled = false);
+ALTER TABLE data SET (sql_stats_automatic_full_collection_enabled = true)
+
+# Insert enough data to trigger a full stats collection.
+statement ok
+INSERT INTO data SELECT a, b, c FROM
+  generate_series(15, 25) AS a(a),
+  generate_series(15, 25) AS b(b),
+  generate_series(15, 25) AS c(c)
+
+query TTIIIT colnames,retry
+SELECT DISTINCT ON (statistics_name, column_names) statistics_name, column_names, row_count, distinct_count, null_count, partial_predicate
+FROM [SHOW STATISTICS FOR TABLE data] ORDER BY statistics_name, column_names, created DESC
+----
+statistics_name  column_names  row_count  distinct_count  null_count  partial_predicate
+__auto__         {a}           2347       22              0           NULL
+__auto__         {a,b}         2347       225             0           NULL
+__auto__         {a,b,c}       2347       2347            0           NULL
+__auto__         {b}           2347       25              0           NULL
+__auto__         {c}           2347       25              0           NULL
+__auto__         {d}           2347       4               1335        NULL