Merge #143616

143616: kvserver: add kv.closed_timestamp.lead_for_global_reads_auto_tune.enabled r=arulajmani a=wenyihu6

This commit adds kv.closed_timestamp.lead_for_global_reads_auto_tune.enabled.

When enabled, ranges which are configured to serve global reads are assigned
policies based on observed latency between their leaseholder and furthest
follower. This dynamic adjustment helps optimize closed timestamp lead times. If
no latency data is available, falls back to default lead times. Note that the
LeadForGlobalReadsOverride setting takes precedence if set.

In addition, this commit changes the description for
kv.closed_timestamp.lead_for_global_reads_auto_tune_interval since we changed
the design, and the system now refreshes policies on every leaseholders
periodically regardless of whether auto-tuning is enabled. This commit
also changes this setting to be reserved rather than public.

Epic: none
Release note: none


Co-authored-by: wenyihu6 <[email protected]>

Author: craig[bot]

docs/generated/settings/settings-for-tenants.txt

@@ -78,7 +78,7 @@ feature.stats.enabled	boolean	true	set to true to enable CREATE STATISTICS/ANALY
 jobs.retention_time	duration	336h0m0s	the amount of time for which records for completed jobs are retained	application
 kv.bulk_sst.target_size	byte size	16 MiB	target size for SSTs emitted from export requests; export requests (i.e. BACKUP) may buffer up to the sum of kv.bulk_sst.target_size and kv.bulk_sst.max_allowed_overage in memory	system-visible
 kv.closed_timestamp.follower_reads.enabled (alias: kv.closed_timestamp.follower_reads_enabled)	boolean	true	allow (all) replicas to serve consistent historical reads based on closed timestamp information	system-visible
-kv.closed_timestamp.lead_for_global_reads_auto_tune_interval	duration	5m0s	interval at which the system updates closed timestamp policies for ranges based on observed network latency. If non-zero, global-read ranges are periodically assigned closed timestamp policies based on their leaseholder-to-furthest-follower latencies, affecting their closed timestamp lead times. If kv.closed_timestamp.lead_for_global_reads_override is set, it takes precedence over the auto-tuned value. Set to 0 to disable auto-tuning and use default lead time for global read ranges without using observed network latency.	system-visible
+kv.closed_timestamp.lead_for_global_reads_auto_tune.enabled	boolean	false	if enabled, observed network latency between leaseholders and their furthest follower will be used to adjust closed timestamp policies for rangesranges configured to serve global reads. kv.closed_timestamp.lead_for_global_reads_override takes precedence if set.	system-visible
 kv.closed_timestamp.lead_for_global_reads_override	duration	0s	if nonzero, overrides the lead time that global_read ranges use to publish closed timestamps	system-visible
 kv.closed_timestamp.side_transport_interval	duration	200ms	the interval at which the closed timestamp side-transport attempts to advance each range's closed timestamp; set to 0 to disable the side-transport	system-visible
 kv.closed_timestamp.target_duration	duration	3s	if nonzero, attempt to provide closed timestamp notifications for timestamps trailing cluster time by approximately this duration	system-visible

docs/generated/settings/settings.html

@@ -94,7 +94,7 @@
 <tr><td><div id="setting-kv-bulk-sst-max-allowed-overage" class="anchored"><code>kv.bulk_sst.max_allowed_overage</code></div></td><td>byte size</td><td><code>64 MiB</code></td><td>if positive, allowed size in excess of target size for SSTs from export requests; export requests (i.e. BACKUP) may buffer up to the sum of kv.bulk_sst.target_size and kv.bulk_sst.max_allowed_overage in memory</td><td>Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-kv-bulk-sst-target-size" class="anchored"><code>kv.bulk_sst.target_size</code></div></td><td>byte size</td><td><code>16 MiB</code></td><td>target size for SSTs emitted from export requests; export requests (i.e. BACKUP) may buffer up to the sum of kv.bulk_sst.target_size and kv.bulk_sst.max_allowed_overage in memory</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-kv-closed-timestamp-follower-reads-enabled" class="anchored"><code>kv.closed_timestamp.follower_reads.enabled<br />(alias: kv.closed_timestamp.follower_reads_enabled)</code></div></td><td>boolean</td><td><code>true</code></td><td>allow (all) replicas to serve consistent historical reads based on closed timestamp information</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
-<tr><td><div id="setting-kv-closed-timestamp-lead-for-global-reads-auto-tune-interval" class="anchored"><code>kv.closed_timestamp.lead_for_global_reads_auto_tune_interval</code></div></td><td>duration</td><td><code>5m0s</code></td><td>interval at which the system updates closed timestamp policies for ranges based on observed network latency. If non-zero, global-read ranges are periodically assigned closed timestamp policies based on their leaseholder-to-furthest-follower latencies, affecting their closed timestamp lead times. If kv.closed_timestamp.lead_for_global_reads_override is set, it takes precedence over the auto-tuned value. Set to 0 to disable auto-tuning and use default lead time for global read ranges without using observed network latency.</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
+<tr><td><div id="setting-kv-closed-timestamp-lead-for-global-reads-auto-tune-enabled" class="anchored"><code>kv.closed_timestamp.lead_for_global_reads_auto_tune.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>if enabled, observed network latency between leaseholders and their furthest follower will be used to adjust closed timestamp policies for rangesranges configured to serve global reads. kv.closed_timestamp.lead_for_global_reads_override takes precedence if set.</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-kv-closed-timestamp-lead-for-global-reads-override" class="anchored"><code>kv.closed_timestamp.lead_for_global_reads_override</code></div></td><td>duration</td><td><code>0s</code></td><td>if nonzero, overrides the lead time that global_read ranges use to publish closed timestamps</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-kv-closed-timestamp-side-transport-interval" class="anchored"><code>kv.closed_timestamp.side_transport_interval</code></div></td><td>duration</td><td><code>200ms</code></td><td>the interval at which the closed timestamp side-transport attempts to advance each range&#39;s closed timestamp; set to 0 to disable the side-transport</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-kv-closed-timestamp-target-duration" class="anchored"><code>kv.closed_timestamp.target_duration</code></div></td><td>duration</td><td><code>3s</code></td><td>if nonzero, attempt to provide closed timestamp notifications for timestamps trailing cluster time by approximately this duration</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>

pkg/kv/kvserver/closedts/policy.go

@@ -16,9 +16,10 @@ const (
 	defaultMaxNetworkRTT = 150 * time.Millisecond
 )
 
-// computeLeadTimeForGlobalReads calculates how far ahead of the current time a node should
-// publish closed timestamps to ensure that followers can serve global reads. It accounts for
-// network latency, clock offset, and both Raft and side-transport propagation delays.
+// computeLeadTimeForGlobalReads calculates how far ahead of the current time a
+// node should publish closed timestamps to ensure that followers can serve
+// global reads. It accounts for network latency, clock offset, and both Raft
+// and side-transport propagation delays.
 func computeLeadTimeForGlobalReads(
 	networkRTT time.Duration, maxClockOffset time.Duration, sideTransportCloseInterval time.Duration,
 ) time.Duration {

pkg/kv/kvserver/closedts/setting.go

@@ -50,23 +50,30 @@ var LeadForGlobalReadsOverride = settings.RegisterDurationSetting(
 	settings.WithPublic,
 )
 
-// LeadForGlobalReadsAutoTuneInterval controls how frequently the system adjusts
-// closed timestamp policies for ranges based on network latency measurements.
-// When enabled, ranges are assigned policies based on observed latency between
-// their leaseholder and furthest follower. This dynamic adjustment helps
-// optimize closed timestamp lead times. If no latency data is available, falls
-// back to default lead times. Note that the LeadForGlobalReadsOverride setting
-// takes precedence if set.
-var LeadForGlobalReadsAutoTuneInterval = settings.RegisterDurationSetting(
+// RangeClosedTimestampPolicyRefreshInterval controls how frequently the system
+// refreshes policies on leaseholders of nodes. These policies are refreshed
+// regardless of whether auto-tuning is enabled.
+var RangeClosedTimestampPolicyRefreshInterval = settings.RegisterDurationSetting(
 	settings.SystemVisible,
-	"kv.closed_timestamp.lead_for_global_reads_auto_tune_interval",
-	"interval at which the system updates closed timestamp policies for ranges based on observed network latency. "+
-		"If non-zero, global-read ranges are periodically assigned closed timestamp policies based on their "+
-		"leaseholder-to-furthest-follower latencies, affecting their closed timestamp lead times. If "+
-		"kv.closed_timestamp.lead_for_global_reads_override is set, "+
-		"it takes precedence over the auto-tuned value. Set to 0 to disable auto-tuning and use default lead time for "+
-		"global read ranges without using observed network latency.",
+	"kv.closed_timestamp.policy_refresh_interval",
+	"interval at which the system refreshes closed timestamp policies for leaseholders",
 	5*time.Minute,
 	settings.NonNegativeDuration,
+)
+
+// LeadForGlobalReadsAutoTuneEnabled determines whether ranges configured to
+// serve global reads would be assigned policies based on observed latency
+// between their leaseholder and furthest follower. This dynamic adjustment
+// helps optimize closed timestamp lead times. If no latency data is available,
+// falls back to default lead times. Note that the LeadForGlobalReadsOverride
+// setting takes precedence if set.
+var LeadForGlobalReadsAutoTuneEnabled = settings.RegisterBoolSetting(
+	settings.SystemVisible,
+	"kv.closed_timestamp.lead_for_global_reads_auto_tune.enabled",
+	"if enabled, observed network latency between leaseholders and their "+
+		"furthest follower will be used to adjust closed timestamp policies for ranges"+
+		"ranges configured to serve global reads. "+
+		"kv.closed_timestamp.lead_for_global_reads_override takes precedence if set.",
+	false,
 	settings.WithPublic,
 )