Merge #144704 #144900 #144969 #144987

144704: kvserver: refresh ReplicaUnavailableErrors on the leaderlessWatcher  r=arulajmani a=arulajmani

Previously, we would construct a ReplicaUnavailableError on the
LeaderlessWatcher once (using an empty descriptor, which is rather
hilarious) and never updated the error. That meant that the returned
error wouldn't reflect the current state of the RangeDescriptor, which
made it hard to make sense of the shape of the unavailability.

This patch fixes the issue by refreshing the cached error on every
available -> unavailable state transition in the leaderlessWatcher.

Closes #144639

Release note: None

144900: sql: optimize SHOW CREATE TABLE performance with many schema objects r=fqazi a=fqazi

Previously, SHOW CREATE TABLE queried the crdb_internal.zones table to extract the zone configuration. This could be slow with a large number of objects because the subquery needed to scan the entirety of crdb_internal.zones, which would do one round-trip per zone config (to fetch descriptors). This patch optimizes crdb_internal.zones to fetch all required descriptors in a single request, instead of performing a round trip for each descriptor. Additionally, this patch adds a new BenchmarkORMQueries test in the rttanalysisccl package, configured for multi-region testing.

Fixes: #141827

Release note (bug fix): Improve slow SHOW CREATE TABLE on multi-region
databases with large number of objects

144969: go.mod: bump Pebble to c70ddd4f9c9e r=RaduBerinde a=jbowens

Changes:

 * [`c70ddd4f`](cockroachdb/pebble@c70ddd4f) db: test replica removal of snapshot sstable
 * [`c5327549`](cockroachdb/pebble@c5327549) manifest,record: disambiguate unexpected EOFs
 * [`57c482fb`](cockroachdb/pebble@57c482fb) db: run score-based compactions before manual compactions
 * [`7ed6738a`](cockroachdb/pebble@7ed6738a) go.mod: update axisds
 * [`a7c678d0`](cockroachdb/pebble@a7c678d0) db: avoid use of internal type in ValueSeparationPolicy
 * [`30569643`](cockroachdb/pebble@30569643) db: export FormatExperimentalValueSeparation
 * [`c25e82e6`](cockroachdb/pebble@c25e82e6) tool: add blob handle print mode to tools
 * [`6816ec8c`](cockroachdb/pebble@6816ec8c) manifest: rename AddedTablesByFileNum to AllAddedTables
 * [`e589e571`](cockroachdb/pebble@e589e571) base: introduce TableNum
 * [`b99c8d26`](cockroachdb/pebble@b99c8d26) db: add TestLargeKeys
 * [`aa0446ac`](cockroachdb/pebble@aa0446ac) objstorageprovider: support blob files
 * [`243d3baf`](cockroachdb/pebble@243d3baf) manifest: remove redundant smallest, largest bounds in TableMetadata
 * [`decdc6ff`](cockroachdb/pebble@decdc6ff) db: deflake TestCheckpoint
 * [`6e324c22`](cockroachdb/pebble@6e324c22) db: include blob files in checkpoints
 * [`0ef24272`](cockroachdb/pebble@0ef24272) db: fix TestCompaction/value_separation merge skew

Release note: none.
Epic: none.

144987: changefeedccl: max changefeed.resolved_timestamp.granularity to 10s r=andyyang890 a=rharding6373

changefeed.resolved_timestamp.granularity was recently made metamorphic, but is causing test flakiness due to timeouts when it is >10s. This PR reduces the max to 10s.

Epic: none
Fixes: #144956
Fixes: #144948
Fixes: #144934

Release note: none

Co-authored-by: Arul Ajmani <[email protected]>
Co-authored-by: Faizan Qazi <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>
Co-authored-by: rharding6373 <[email protected]>

Author: 5 people

DEPS.bzl

@@ -1915,10 +1915,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "177126dda72103d057a70dbe645a344506b9a0f4a88b2d3ace9f5880c3af05db",
-        strip_prefix = "github.com/cockroachdb/[email protected]20250417234720-091ed497241b",
+        sha256 = "8feaf8e2bf545b0e31247314d9b4f894b13e08aedc9bb425ae27966144e6c234",
+        strip_prefix = "github.com/cockroachdb/[email protected]20250423160643-c70ddd4f9c9e",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250417234720-091ed497241b.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250423160643-c70ddd4f9c9e.zip",
         ],
     )
     go_repository(
@@ -7986,10 +7986,10 @@ def go_deps():
         name = "com_github_raduberinde_axisds",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/RaduBerinde/axisds",
-        sha256 = "53a2821f7d7cc76e91278e457d5f58bc5a856c7968d4170435dac86f2d003a87",
-        strip_prefix = "github.com/RaduBerinde/[email protected]20250405232732-ecb85bedf677",
+        sha256 = "e943d3ac26630cf9e337483450ad9db7f43b6a0372a79d5a691aad8c3c92f9e3",
+        strip_prefix = "github.com/RaduBerinde/[email protected]20250419182453-5135a0650657",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/axisds/com_github_raduberinde_axisds-v0.0.0-20250405232732-ecb85bedf677.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/axisds/com_github_raduberinde_axisds-v0.0.0-20250419182453-5135a0650657.zip",
         ],
     )
     go_repository(
@@ -8002,6 +8002,16 @@ def go_deps():
             "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/btree/com_github_raduberinde_btree-v1.0.2-0.20250415192849-8beea01764ce.zip",
         ],
     )
+    go_repository(
+        name = "com_github_raduberinde_btreemap",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/RaduBerinde/btreemap",
+        sha256 = "81b6e0391c0bceadab0bc56184a7ed25dfe81daaa04916900f66b55f34590f6b",
+        strip_prefix = "github.com/RaduBerinde/[email protected]",
+        urls = [
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/btreemap/com_github_raduberinde_btreemap-v0.0.0-20250419174037-3d62b7205d54.zip",
+        ],
+    )
     go_repository(
         name = "com_github_rcrowley_go_metrics",
         build_file_proto_mode = "disable_global",

build/bazelutil/distdir_files.bzl

@@ -207,8 +207,9 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/PuerkitoBio/goquery/com_github_puerkitobio_goquery-v1.5.1.zip": "9d5bbc466dc4fac7ad872f69eeb9dcf6ddfd925821c4699226fbdeae117839a2",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/PuerkitoBio/purell/com_github_puerkitobio_purell-v1.1.1.zip": "59e636760d7f2ab41c2f80c1784b1c73d381d44888d1999228dedd634ddcf5ed",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/PuerkitoBio/urlesc/com_github_puerkitobio_urlesc-v0.0.0-20170810143723-de5bf2ad4578.zip": "1793124273dd94e7089e95716d40529bcf70b9e87162d60218f68dde4d6aeb9d",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/axisds/com_github_raduberinde_axisds-v0.0.0-20250405232732-ecb85bedf677.zip": "53a2821f7d7cc76e91278e457d5f58bc5a856c7968d4170435dac86f2d003a87",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/axisds/com_github_raduberinde_axisds-v0.0.0-20250419182453-5135a0650657.zip": "e943d3ac26630cf9e337483450ad9db7f43b6a0372a79d5a691aad8c3c92f9e3",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/btree/com_github_raduberinde_btree-v1.0.2-0.20250415192849-8beea01764ce.zip": "1c845dc6e77f27888fa66c8521df4d1d30f3ccf6892e8c1ba0aab4db24e717c1",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/RaduBerinde/btreemap/com_github_raduberinde_btreemap-v0.0.0-20250419174037-3d62b7205d54.zip": "81b6e0391c0bceadab0bc56184a7ed25dfe81daaa04916900f66b55f34590f6b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/SAP/go-hdb/com_github_sap_go_hdb-v0.14.1.zip": "273de28a254c39e9f24293b864c1d664488e4a5d44d535755a5e5b68ae7eed8d",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/Shopify/goreferrer/com_github_shopify_goreferrer-v0.0.0-20220729165902-8cddb4f5de06.zip": "280a2f55812e8b475cfd9d467a3b3d5859315788e68592a8fc5d6cedadc0503f",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/Shopify/logrus-bugsnag/com_github_shopify_logrus_bugsnag-v0.0.0-20171204204709-577dee27f20d.zip": "a4cc3fa4b7b493b36b96ea035caa7afcf7307b0c4efc5e523a46597e171b95ce",
@@ -365,7 +366,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250417234720-091ed497241b.zip": "177126dda72103d057a70dbe645a344506b9a0f4a88b2d3ace9f5880c3af05db",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250423160643-c70ddd4f9c9e.zip": "8feaf8e2bf545b0e31247314d9b4f894b13e08aedc9bb425ae27966144e6c234",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

go.mod

@@ -138,7 +138,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
-	github.com/cockroachdb/pebble v0.0.0-20250417234720-091ed497241b
+	github.com/cockroachdb/pebble v0.0.0-20250423160643-c70ddd4f9c9e
 	github.com/cockroachdb/redact v1.1.6
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b
@@ -289,7 +289,8 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/RaduBerinde/axisds v0.0.0-20250405232732-ecb85bedf677 // indirect
+	github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect
+	github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect
 	github.com/abbot/go-http-auth v0.4.1-0.20181019201920-860ed7f246ff // indirect
 	github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect
 	github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect

go.sum

@@ -266,10 +266,12 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/RaduBerinde/axisds v0.0.0-20250405232732-ecb85bedf677 h1:NofOMIO/Z3301wDc9gIM/M0/+YjvOYSvUe7+30bkxkA=
-github.com/RaduBerinde/axisds v0.0.0-20250405232732-ecb85bedf677/go.mod h1:YO26VdZg1RVVjEhjmeEJE/4bLnD2mowj5eDiZQSLtlE=
+github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8=
+github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y=
 github.com/RaduBerinde/btree v1.0.2-0.20250415192849-8beea01764ce h1:jmg1o6EkSTLFQ4vdTBzob5AjFBzNH4ELn56iKhMVk48=
 github.com/RaduBerinde/btree v1.0.2-0.20250415192849-8beea01764ce/go.mod h1:1+GykKoq3bIUWDVjNWWxWYblBduYahx5NGepZjgYqvI=
+github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk=
+github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc=
 github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
@@ -593,8 +595,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
 github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
-github.com/cockroachdb/pebble v0.0.0-20250417234720-091ed497241b h1:x0mDX6DRVJphx9xkup4N00L6nDn6K+cSOIUXKvbmeSI=
-github.com/cockroachdb/pebble v0.0.0-20250417234720-091ed497241b/go.mod h1:ONMS0WzY11hEvOyO3Cl6fyfaDduyiz42cYeQAcqHRNQ=
+github.com/cockroachdb/pebble v0.0.0-20250423160643-c70ddd4f9c9e h1:zG2jqS8AgpMbLThyQm3HrKs/yaLV8CORKLn0N4hoGxA=
+github.com/cockroachdb/pebble v0.0.0-20250423160643-c70ddd4f9c9e/go.mod h1:jyGqUZ9jl6TVX9HHi12f0JQ+h+oL75De//O43X898BA=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
 github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/ccl/benchccl/rttanalysisccl/multi_region_bench_test.go

@@ -7,7 +7,9 @@ package rttanalysisccl
 
 import (
 	gosql "database/sql"
+	"fmt"
 	"net/url"
+	"strings"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -69,6 +71,37 @@ CREATE TABLE test42 (p int) LOCALITY REGIONAL BY TABLE IN "us-east2";
 `
 )
 
+// multipleTableFixtures creates multiple tables with different localities.
+func multipleTableFixtures(n int) string {
+	b := strings.Builder{}
+
+	b.WriteString(`CREATE DATABASE test PRIMARY REGION "us-east1" REGIONS "us-east1", "us-east2", "us-east3";
+USE test;`)
+	b.WriteString("BEGIN;\n")
+	b.WriteString("SET LOCAL autocommit_before_ddl = false;\n")
+	for i := 0; i < n; i++ {
+		b.WriteString(fmt.Sprintf("CREATE TABLE test%d (p int)", i))
+
+		locality := i % 5
+
+		switch locality {
+		case 0:
+			b.WriteString(" LOCALITY GLOBAL")
+		case 1:
+			b.WriteString(" LOCALITY REGIONAL BY ROW")
+		case 2:
+			b.WriteString(" LOCALITY REGIONAL BY TABLE IN \"us-east1\"")
+		case 3:
+			b.WriteString(" LOCALITY REGIONAL BY TABLE IN \"us-east2\"")
+		case 4:
+			b.WriteString(" LOCALITY REGIONAL BY TABLE IN \"us-east3\"")
+		}
+		b.WriteString(";\n")
+	}
+	b.WriteString("COMMIT;\n")
+	return b.String()
+}
+
 func BenchmarkAlterRegions(b *testing.B) { reg.Run(b) }
 func init() {
 	reg.Register("AlterRegions", []rttanalysis.RoundTripBenchTestCase{
@@ -238,3 +271,28 @@ CREATE TABLE test (p int) LOCALITY REGIONAL BY ROW;
 		},
 	})
 }
+
+func BenchmarkVirtualTableQueries(b *testing.B) { reg.Run(b) }
+func init() {
+	reg.Register("VirtualTableQueries", []rttanalysis.RoundTripBenchTestCase{
+		{
+			Name:  "select from crdb_internal.zones (10 tables)",
+			Setup: multipleTableFixtures(10),
+			Stmt:  `select * from crdb_internal.zones`,
+			Reset: "DROP DATABASE test",
+		},
+		{
+			Name:  "select from crdb_internal.zones (50 tables)",
+			Setup: multipleTableFixtures(50),
+			Stmt:  `select * from crdb_internal.zones`,
+			Reset: "DROP DATABASE test",
+		},
+
+		{
+			Name:  "select from crdb_internal.zones (100 tables)",
+			Setup: multipleTableFixtures(100),
+			Stmt:  `select * from crdb_internal.zones`,
+			Reset: "DROP DATABASE test",
+		},
+	})
+}

pkg/ccl/benchccl/rttanalysisccl/testdata/benchmark_expectations

@@ -17,3 +17,6 @@ exp,benchmark
 10,AlterTableLocality/alter_from_rbr_to_regional_by_table
 13,AlterTableLocality/alter_from_regional_by_table_to_global
 18,AlterTableLocality/alter_from_regional_by_table_to_rbr
+10,VirtualTableQueries/select_from_crdb_internal.zones_(100_tables)
+10,VirtualTableQueries/select_from_crdb_internal.zones_(10_tables)
+10,VirtualTableQueries/select_from_crdb_internal.zones_(50_tables)

pkg/ccl/changefeedccl/changefeedbase/settings.go

@@ -346,5 +346,5 @@ var Quantize = settings.RegisterDurationSettingWithExplicitUnit(
 	settings.ApplicationLevel,
 	"changefeed.resolved_timestamp.granularity",
 	"the granularity at which changefeed progress is quantized to make tracking more efficient",
-	time.Duration(metamorphic.ConstantWithTestRange("changefeed.resolved_timestamp.granularity", 0, 0, 20))*time.Second,
+	time.Duration(metamorphic.ConstantWithTestRange("changefeed.resolved_timestamp.granularity", 0, 0, 10))*time.Second,
 )

pkg/kv/kvserver/client_replica_test.go

@@ -68,6 +68,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
+	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
@@ -2788,7 +2789,6 @@ func TestLossQuorumCauseLeaderWatcherToSignalUnavailable(t *testing.T) {
 	require.NoError(t, log.SetVModule("replica_range_lease=3,raft=4"))
 
 	ctx := context.Background()
-	manualClock := hlc.NewHybridManualClock()
 	stickyVFSRegistry := fs.NewStickyRegistry()
 	lisReg := listenerutil.NewListenerRegistry()
 	defer lisReg.Close()
@@ -2851,12 +2851,8 @@ func TestLossQuorumCauseLeaderWatcherToSignalUnavailable(t *testing.T) {
 		return nil
 	})
 
-	// Increment the clock by the leaderlessWatcher unavailable threshold.
-	manualClock.Increment(threshold.Nanoseconds())
-
 	// Wait for the leaderlessWatcher to indicate that the range is unavailable.
 	testutils.SucceedsSoon(t, func() error {
-		tc.GetFirstStoreFromServer(t, aliveNodeIdx).LookupReplica(roachpb.RKey(key))
 		if !repl.LeaderlessWatcher.IsUnavailable() {
 			return errors.New("range is still available")
 		}
@@ -2915,6 +2911,77 @@ func TestLossQuorumCauseLeaderWatcherToSignalUnavailable(t *testing.T) {
 	})
 }
 
+// TestLeaderlessWatcherUnavailabilityErrorRefreshedOnUnavailabilityTransition
+// ensures that the leaderless watcher constructs a new error every time it
+// transitions to the unavailable state. In particular, the descriptor used
+// in the error should be the latest descriptor.
+// Serves as a regression test for
+// https://github.com/cockroachdb/cockroach/issues/144639.
+func TestLeaderlessWatcherErrorRefreshedOnUnavailabilityTransition(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+	stopper := stop.NewStopper()
+	defer stopper.Stop(ctx)
+
+	manual := hlc.NewHybridManualClock()
+	st := cluster.MakeTestingClusterSettings()
+	// Set the leaderless threshold to 10 second.
+	kvserver.ReplicaLeaderlessUnavailableThreshold.Override(ctx, &st.SV, 10*time.Second)
+
+	tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{
+		ReplicationMode: base.ReplicationManual,
+		ServerArgs: base.TestServerArgs{
+			Settings: st,
+			Knobs: base.TestingKnobs{
+				Server: &server.TestingKnobs{
+					WallClock: manual,
+				},
+			},
+		},
+	})
+	defer tc.Stopper().Stop(ctx)
+	key := tc.ScratchRange(t)
+	tc.AddVotersOrFatal(t, key, tc.Targets(1)...)
+	repl := tc.GetFirstStoreFromServer(t, 1).LookupReplica(roachpb.RKey(key))
+
+	// The leaderlessWatcher starts off as available.
+	require.False(t, repl.LeaderlessWatcher.IsUnavailable())
+	// Let it know it's leaderless.
+	repl.RefreshLeaderlessWatcherUnavailableStateForTesting(ctx, raft.None, manual.Now(), st)
+	// Even though the replica is leaderless, enough time hasn't passed for it to
+	// be considered unavailable.
+	require.False(t, repl.LeaderlessWatcher.IsUnavailable())
+	// The error should be nil as we're not considered leaderless at this point.
+	require.NoError(t, repl.LeaderlessWatcher.Err())
+	// Let enough time pass.
+	manual.Increment(10 * time.Second.Nanoseconds())
+	repl.RefreshLeaderlessWatcherUnavailableStateForTesting(ctx, raft.None, manual.Now(), st)
+	// Now the replica is considered unavailable.
+	require.True(t, repl.LeaderlessWatcher.IsUnavailable())
+	require.Error(t, repl.LeaderlessWatcher.Err())
+	// Regex to ensure we've got a replica unavailable error with n1 and n2 in the
+	// range descriptor.
+	require.Regexp(t, "replica unavailable.*n1.*n2.*gen=3", repl.LeaderlessWatcher.Err().Error())
+
+	// Next up, let the replica know there's a leader. This should make it
+	// available again.
+	repl.RefreshLeaderlessWatcherUnavailableStateForTesting(ctx, 1, manual.Now(), st)
+	require.False(t, repl.LeaderlessWatcher.IsUnavailable())
+	// Change the range descriptor. Mark it leaderless and let enough time pass
+	// for it to be considered unavailable again.
+	tc.AddVotersOrFatal(t, key, tc.Targets(2)...)
+	repl.RefreshLeaderlessWatcherUnavailableStateForTesting(ctx, raft.None, manual.Now(), st)
+	manual.Increment(10 * time.Second.Nanoseconds())
+	repl.RefreshLeaderlessWatcherUnavailableStateForTesting(ctx, raft.None, manual.Now(), st)
+	// The replica should now be considered unavailable again.
+	require.True(t, repl.LeaderlessWatcher.IsUnavailable())
+	require.Error(t, repl.LeaderlessWatcher.Err())
+	// Ensure that the range descriptor now contains n1, n2, and n3 -- i.e, we're
+	// updating the error with the latest descriptor on the latest transition.
+	require.Regexp(t, "replica unavailable.*n1.*n2.*n3.*gen=5", repl.LeaderlessWatcher.Err().Error())
+}
+
 func TestClearRange(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)