Add a feature gate to use PD's ready API as the readiness probe (pingcap#6282)

fgksgf · liubog2008 · commit 89d7661e092b · 2025-11-20T11:56:37.000+08:00
diff --git a/.github/licenserc.yaml b/.github/licenserc.yaml
@@ -29,4 +29,5 @@ header:
     - '**/OWNERS'
     - OWNERS_ALIASES
     - .cursorignore
+    - .claude/**
   comment: on-failure
diff --git a/.gitignore b/.gitignore
@@ -4,4 +4,6 @@ output
 coverage.txt
 .vscode/
 .cursorignore
-.DS_Store
+.DS_Store
+CLAUDE.md
+.claude/
diff --git a/Makefile b/Makefile
@@ -239,7 +239,7 @@ bin/kind:
 .PHONY: bin/license-eye
 LICENSE_EYE = $(BIN_DIR)/license-eye
 bin/license-eye:
-	if [ ! -f $(LICENSE_EYE) ]; then $(ROOT)/hack/download.sh go_install $(LICENSE_EYE) github.com/apache/skywalking-eyes/cmd/license-eye v0.6.0; fi
+	if [ ! -f $(LICENSE_EYE) ]; then $(ROOT)/hack/download.sh go_install $(LICENSE_EYE) github.com/apache/skywalking-eyes/cmd/license-eye 049742de2276515409e3109ca2a91934053e080d; fi
 
 .PHONY: bin/ginkgo
 GINKGO = $(BIN_DIR)/ginkgo
diff --git a/api/meta/v1alpha1/feature.go b/api/meta/v1alpha1/feature.go
@@ -18,7 +18,7 @@ package v1alpha1
 // NOTE(liubo02): +enum is not supported now, we have to add all enum into comments
 // NOTE(liubo02): It's supported by https://github.com/kubernetes-sigs/controller-tools/pull/1179
 //
-// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe
+// +kubebuilder:validation:Enum=FeatureModification;VolumeAttributesClass;DisablePDDefaultReadinessProbe;UsePDReadyAPI
 // +enum
 type Feature string
 
@@ -56,4 +56,9 @@ const (
 	// It's not useful and will print so many warn logs in PD's stdout/stderr
 	DisablePDDefaultReadinessProbe      Feature      = "DisablePDDefaultReadinessProbe"
 	DisablePDDefaultReadinessProbeStage FeatureStage = FeatureStageAlpha
+
+	// UsePDReadyAPI means use PD's /ready API as the readiness probe.
+	// It requires PD v8.5.2 or later.
+	UsePDReadyAPI      Feature      = "UsePDReadyAPI"
+	UsePDReadyAPIStage FeatureStage = FeatureStageAlpha
 )
diff --git a/go.work.sum b/go.work.sum
@@ -491,6 +491,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE=
 github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/ionos-cloud/sdk-go/v6 v6.1.11/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
diff --git a/hack/lib/e2e.sh b/hack/lib/e2e.sh
@@ -353,7 +353,7 @@ function e2e::run() {
 
 function e2e::run_upgrade() {
     e2e::install_old_version
-    $GINKGO -v -r --tags=upgrade_e2e --timeout=1h --randomize-all --randomize-suites --fail-on-empty --race --trace "$ROOT/tests/e2e/upgrade"
+    $GINKGO -v -r --tags=upgrade_e2e --timeout=1h --randomize-all --randomize-suites --fail-on-empty --race --trace --flake-attempts=2 "$ROOT/tests/e2e/upgrade"
 }
 
 function e2e::prepare() {
diff --git a/hack/lib/vars.sh b/hack/lib/vars.sh
@@ -59,11 +59,11 @@ readonly V_KIND_CLUSTER=${V_KIND_CLUSTER:-"tidb-operator"}
 # V_DEPLOY_NAMESPACE defines namespace of deploy, the default value is tidb-admin
 readonly V_DEPLOY_NAMESPACE=${V_DEPLOY_NAMESPACE:-"tidb-admin"}
 
-# V_TIDB_CLUSTER_VERSION defines version of TiDB cluster, the default value is v8.2.0
-readonly V_TIDB_CLUSTER_VERSION=${V_TIDB_CLUSTER_VERSION:-"v8.2.0"}
+# V_TIDB_CLUSTER_VERSION defines version of TiDB cluster, the default value is v8.5.2
+readonly V_TIDB_CLUSTER_VERSION=${V_TIDB_CLUSTER_VERSION:-"v8.5.2"}
 
-# V_TIDB_CLUSTER_VERSION_PREV defines a older version of TiDB cluster, the default value is v8.1.0
-readonly V_TIDB_CLUSTER_VERSION_PREV=${V_TIDB_CLUSTER_VERSION_PREV:-"v8.1.0"}
+# V_TIDB_CLUSTER_VERSION_PREV defines a older version of TiDB cluster, the default value is v8.5.0
+readonly V_TIDB_CLUSTER_VERSION_PREV=${V_TIDB_CLUSTER_VERSION_PREV:-"v8.5.0"}
 
 # V_KUBE_VERSION defines default test version of kubernetes
 readonly V_KUBE_VERSION=${V_KUBE_VERSION:-"v1.31.0"}
diff --git a/manifests/crd/core.pingcap.com_clusters.yaml b/manifests/crd/core.pingcap.com_clusters.yaml
@@ -89,6 +89,7 @@ spec:
                       - FeatureModification
                       - VolumeAttributesClass
                       - DisablePDDefaultReadinessProbe
+                      - UsePDReadyAPI
                       type: string
                   required:
                   - name
@@ -267,6 +268,7 @@ spec:
                       - FeatureModification
                       - VolumeAttributesClass
                       - DisablePDDefaultReadinessProbe
+                      - UsePDReadyAPI
                       type: string
                     stage:
                       type: string
diff --git a/manifests/crd/core.pingcap.com_pdgroups.yaml b/manifests/crd/core.pingcap.com_pdgroups.yaml
@@ -107,6 +107,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_pds.yaml b/manifests/crd/core.pingcap.com_pds.yaml
@@ -83,6 +83,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_schedulergroups.yaml b/manifests/crd/core.pingcap.com_schedulergroups.yaml
@@ -96,6 +96,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_schedulers.yaml b/manifests/crd/core.pingcap.com_schedulers.yaml
@@ -83,6 +83,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_ticdcgroups.yaml b/manifests/crd/core.pingcap.com_ticdcgroups.yaml
@@ -93,6 +93,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_ticdcs.yaml b/manifests/crd/core.pingcap.com_ticdcs.yaml
@@ -80,6 +80,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tidbgroups.yaml b/manifests/crd/core.pingcap.com_tidbgroups.yaml
@@ -96,6 +96,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tidbs.yaml b/manifests/crd/core.pingcap.com_tidbs.yaml
@@ -81,6 +81,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiflashes.yaml b/manifests/crd/core.pingcap.com_tiflashes.yaml
@@ -85,6 +85,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiflashgroups.yaml b/manifests/crd/core.pingcap.com_tiflashgroups.yaml
@@ -95,6 +95,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tikvgroups.yaml b/manifests/crd/core.pingcap.com_tikvgroups.yaml
@@ -96,6 +96,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tikvs.yaml b/manifests/crd/core.pingcap.com_tikvs.yaml
@@ -85,6 +85,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiproxies.yaml b/manifests/crd/core.pingcap.com_tiproxies.yaml
@@ -81,6 +81,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/manifests/crd/core.pingcap.com_tiproxygroups.yaml b/manifests/crd/core.pingcap.com_tiproxygroups.yaml
@@ -96,6 +96,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tsogroups.yaml b/manifests/crd/core.pingcap.com_tsogroups.yaml
@@ -96,6 +96,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               replicas:
diff --git a/manifests/crd/core.pingcap.com_tsos.yaml b/manifests/crd/core.pingcap.com_tsos.yaml
@@ -82,6 +82,7 @@ spec:
                   - FeatureModification
                   - VolumeAttributesClass
                   - DisablePDDefaultReadinessProbe
+                  - UsePDReadyAPI
                   type: string
                 type: array
               image:
diff --git a/pkg/configs/pd/config.go b/pkg/configs/pd/config.go
@@ -75,7 +75,7 @@ func (c *Config) Overlay(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, peers []*v1
 
 	c.Name = pd.Name
 	c.ClientUrls = getClientURLs(pd, scheme)
-	c.AdvertiseClientUrls = getAdvertiseClientURLs(pd, scheme)
+	c.AdvertiseClientUrls = GetAdvertiseClientURLs(pd, scheme)
 	c.PeerUrls = getPeerURLs(pd, scheme)
 	c.AdvertisePeerUrls = getAdvertisePeerURLs(pd, scheme)
 
@@ -170,7 +170,7 @@ func getClientURLs(pd *v1alpha1.PD, scheme string) string {
 	return fmt.Sprintf("%s://[::]:%d", scheme, coreutil.PDClientPort(pd))
 }
 
-func getAdvertiseClientURLs(pd *v1alpha1.PD, scheme string) string {
+func GetAdvertiseClientURLs(pd *v1alpha1.PD, scheme string) string {
 	ns := pd.Namespace
 	if ns == "" {
 		ns = corev1.NamespaceDefault
diff --git a/pkg/controllers/br/manager/util/remote_test.go b/pkg/controllers/br/manager/util/remote_test.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -291,6 +292,9 @@ func TestStorageBackendBasic(t *testing.T) {
 }
 
 func TestStorageBackendBatchDeleteObjects(t *testing.T) {
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping test on macOS due to gomonkey instability")
+	}
 	g := gomega.NewGomegaWithT(t)
 
 	type testcase struct {
diff --git a/pkg/controllers/pd/tasks/ctx.go b/pkg/controllers/pd/tasks/ctx.go
@@ -17,8 +17,12 @@ package tasks
 import (
 	"context"
 
+	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
+	coreutil "github.com/pingcap/tidb-operator/pkg/apiutil/core/v1alpha1"
+	pdcfg "github.com/pingcap/tidb-operator/pkg/configs/pd"
 	"github.com/pingcap/tidb-operator/pkg/timanager"
 	pdm "github.com/pingcap/tidb-operator/pkg/timanager/pd"
 	"github.com/pingcap/tidb-operator/pkg/utils/task/v3"
@@ -38,7 +42,7 @@ type ReconcileContext struct {
 }
 
 func TaskContextInfoFromPD(state *ReconcileContext, cm pdm.PDClientManager) task.Task {
-	return task.NameTaskFunc("ContextInfoFromPD", func(_ context.Context) task.Result {
+	return task.NameTaskFunc("ContextInfoFromPD", func(ctx context.Context) task.Result {
 		ck := state.Cluster()
 		pc, ok := cm.Get(timanager.PrimaryKey(ck.Namespace, ck.Name))
 		if !ok {
@@ -53,7 +57,8 @@ func TaskContextInfoFromPD(state *ReconcileContext, cm pdm.PDClientManager) task
 
 		state.Initialized = true
 
-		m, err := pc.Members().Get(state.PD().Name)
+		pd := state.PD()
+		m, err := pc.Members().Get(pd.Name)
 		if err != nil {
 			if errors.IsNotFound(err) {
 				return task.Complete().With("context without member info is completed, pd is not initialized")
@@ -65,13 +70,20 @@ func TaskContextInfoFromPD(state *ReconcileContext, cm pdm.PDClientManager) task
 		state.MemberID = m.ID
 		state.IsLeader = m.IsLeader
 
+		logger := logr.FromContextOrDiscard(ctx)
+		ready, err := state.PDClient.Underlay().GetMemberReady(ctx, getPDURL(ck, pd), pd.Spec.Version)
+		if err != nil {
+			// Do not return error here, because the pd pod may not be created yet.
+			logger.Error(err, "failed to get member ready")
+		}
+
 		// set available and trust health info only when member info is valid
-		if !m.Invalid && m.Health {
+		if !m.Invalid && m.Health && ready {
 			state.SetHealthy()
 			return task.Complete().With("pd is ready")
 		}
 
-		return task.Wait().With("pd is unready, invalid: %v, health: %v", m.Invalid, m.Health)
+		return task.Wait().With("pd is unready, invalid: %v, health: %v, ready: %v", m.Invalid, m.Health, ready)
 	})
 }
 
@@ -81,3 +93,11 @@ func CondPDClientIsNotRegisterred(state *ReconcileContext) task.Condition {
 		return state.PDClient == nil || !state.PDClient.HasSynced()
 	})
 }
+
+func getPDURL(cluster *v1alpha1.Cluster, pd *v1alpha1.PD) string {
+	scheme := "http"
+	if coreutil.IsTLSClusterEnabled(cluster) {
+		scheme = "https"
+	}
+	return pdcfg.GetAdvertiseClientURLs(pd, scheme)
+}
diff --git a/pkg/controllers/pd/tasks/pod.go b/pkg/controllers/pd/tasks/pod.go
@@ -17,8 +17,10 @@ package tasks
 import (
 	"context"
 	"fmt"
+	"path"
 	"path/filepath"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +30,7 @@ import (
 	metav1alpha1 "github.com/pingcap/tidb-operator/api/v2/meta/v1alpha1"
 	coreutil "github.com/pingcap/tidb-operator/pkg/apiutil/core/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client"
+	"github.com/pingcap/tidb-operator/pkg/compatibility"
 	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/image"
 	"github.com/pingcap/tidb-operator/pkg/overlay"
@@ -247,6 +250,13 @@ func newPod(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, g features.Gates, cluste
 		pod.Spec.Containers[0].ReadinessProbe = buildPDReadinessProbe(coreutil.PDClientPort(pd))
 	}
 
+	if g.Enabled(metav1alpha1.UsePDReadyAPI) {
+		version := semver.MustParse(pd.Spec.Version)
+		if compatibility.Check(version, compatibility.PDReadyAPI) {
+			pod.Spec.Containers[0].ReadinessProbe = buildPDReadinessProbeWithReadyAPI(cluster, coreutil.PDClientPort(pd))
+		}
+	}
+
 	if pd.Spec.Overlay != nil {
 		overlay.OverlayPod(pod, pd.Spec.Overlay.Pod)
 	}
@@ -265,3 +275,40 @@ func buildPDReadinessProbe(port int32) *corev1.Probe {
 		InitialDelaySeconds: defaultReadinessProbeInitialDelaySeconds,
 	}
 }
+
+func buildPDReadinessProbeWithReadyAPI(cluster *v1alpha1.Cluster, port int32) *corev1.Probe {
+	tlsClusterEnabled := coreutil.IsTLSClusterEnabled(cluster)
+
+	scheme := "http"
+	if tlsClusterEnabled {
+		scheme = "https"
+	}
+
+	readinessURL := fmt.Sprintf("%s://127.0.0.1:%d/pd/api/v2/ready", scheme, port)
+	var command []string
+	command = append(command, "curl", readinessURL,
+		// Fail silently (no output at all) on server errors
+		// without this if the server return 500, the exist code will be 0
+		// and probe is success.
+		"--fail",
+		// Silent mode, only print error
+		"-sS",
+		// follow 301 or 302 redirect
+		"--location")
+
+	if tlsClusterEnabled {
+		cacert := path.Join(v1alpha1.DirPathClusterTLSPD, corev1.ServiceAccountRootCAKey)
+		cert := path.Join(v1alpha1.DirPathClusterTLSPD, corev1.TLSCertKey)
+		key := path.Join(v1alpha1.DirPathClusterTLSPD, corev1.TLSPrivateKeyKey)
+		command = append(command, "--cacert", cacert, "--cert", cert, "--key", key)
+	}
+
+	return &corev1.Probe{
+		InitialDelaySeconds: defaultReadinessProbeInitialDelaySeconds,
+		ProbeHandler: corev1.ProbeHandler{
+			Exec: &corev1.ExecAction{
+				Command: command,
+			},
+		},
+	}
+}
diff --git a/pkg/pdapi/v1/client.go b/pkg/pdapi/v1/client.go
diff --git a/pkg/pdapi/v1/client_test.go b/pkg/pdapi/v1/client_test.go
diff --git a/pkg/pdapi/v1/mock_generated.go b/pkg/pdapi/v1/mock_generated.go
diff --git a/tests/e2e/cluster/cluster.go b/tests/e2e/cluster/cluster.go
diff --git a/tests/e2e/data/cluster.go b/tests/e2e/data/cluster.go
diff --git a/tests/e2e/data/data.go b/tests/e2e/data/data.go
diff --git a/tests/e2e/pd/pd.go b/tests/e2e/pd/pd.go
diff --git a/tests/e2e/upgrade/upgrade.go b/tests/e2e/upgrade/upgrade.go
diff --git a/tests/e2e/utils/data/cluster.go b/tests/e2e/utils/data/cluster.go

Original file line number	Diff line number	Diff line change
`@@ -353,7 +353,7 @@ function e2e::run() {`
`353`	`353`
`354`	`354`	`function e2e::run_upgrade() {`
`355`	`355`	`e2e::install_old_version`
`356`		`- $GINKGO -v -r --tags=upgrade_e2e --timeout=1h --randomize-all --randomize-suites --fail-on-empty --race --trace "$ROOT/tests/e2e/upgrade"`
	`356`	`+ $GINKGO -v -r --tags=upgrade_e2e --timeout=1h --randomize-all --randomize-suites --fail-on-empty --race --trace --flake-attempts=2 "$ROOT/tests/e2e/upgrade"`
`357`	`357`	`}`
`358`	`358`
`359`	`359`	`function e2e::prepare() {`