feat(crd): support applying crd by operator (#6644)

Author: liubog2008

charts/tidb-operator/templates/deployment.yaml

@@ -44,11 +44,14 @@ spec:
       - command:
         - /tidb-operator
         args:
-          - --backup-manager-image={{ template "image" (tuple .Values.backupManager.image $.Chart.AppVersion) }}
-          - --default-prestop-checker-image={{ template "image" (tuple .Values.prestopChecker.image $.Chart.AppVersion) }}
-          - --default-resource-syncer-image={{ template "image" (tuple .Values.resourceSyncer.image $.Chart.AppVersion) }}
+        - --backup-manager-image={{ template "image" (tuple .Values.backupManager.image $.Chart.AppVersion) }}
+        - --default-prestop-checker-image={{ template "image" (tuple .Values.prestopChecker.image $.Chart.AppVersion) }}
+        - --default-resource-syncer-image={{ template "image" (tuple .Values.resourceSyncer.image $.Chart.AppVersion) }}
+        {{- if eq .Values.skipCRDsManagement true }}
+        - --skip-crds-management
+        {{- end }}
         {{- if .Values.operator.extraArgs -}}
-        {{ toYaml .Values.operator.extraArgs | nindent 10 }}
+        {{ toYaml .Values.operator.extraArgs | nindent 8 }}
         {{- end }}
         image: "{{ template "image" (tuple .Values.operator.image $.Chart.AppVersion) }}"
         imagePullPolicy: {{ .Values.operator.pullPolicy | default "IfNotPresent" }}

charts/tidb-operator/templates/rbac.yaml

@@ -54,6 +54,12 @@ rules:
   - apiGroups: ["br.pingcap.com"]
     resources: ["*"]
     verbs: ["*"]
+  {{- if eq .Values.skipCRDsManagement false }}
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "create", "update", "patch"]
+    resourceNames: {{ toYaml .Values.crds | nindent 4 }}
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

charts/tidb-operator/values.yaml

@@ -16,8 +16,8 @@ operator:
     # digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
 
   pullPolicy: IfNotPresent
-# # REF: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
-#  priorityClassName: system-cluster-critical
+  # # REF: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  # priorityClassName: system-cluster-critical
   resources:
     requests:
       cpu: 80m
@@ -90,3 +90,37 @@ rbac:
 # Webhook configuration
 webhook:
   create: true
+
+# If false, manage crds by operator directly.
+# If true, operator will not check and apply crds.
+skipCRDsManagement: false
+
+# DON'T EDIT IT
+crds:
+- backups.br.pingcap.com
+- backupschedules.br.pingcap.com
+- compactbackups.br.pingcap.com
+- restores.br.pingcap.com
+- tibrgcs.br.pingcap.com
+- tibrs.br.pingcap.com
+- clusters.core.pingcap.com
+- pdgroups.core.pingcap.com
+- pds.core.pingcap.com
+- schedulergroups.core.pingcap.com
+- schedulers.core.pingcap.com
+- schedulinggroups.core.pingcap.com
+- schedulings.core.pingcap.com
+- ticdcgroups.core.pingcap.com
+- ticdcs.core.pingcap.com
+- tidbgroups.core.pingcap.com
+- tidbs.core.pingcap.com
+- tiflashes.core.pingcap.com
+- tiflashgroups.core.pingcap.com
+- tikvgroups.core.pingcap.com
+- tikvs.core.pingcap.com
+- tikvworkergroups.core.pingcap.com
+- tikvworkers.core.pingcap.com
+- tiproxies.core.pingcap.com
+- tiproxygroups.core.pingcap.com
+- tsogroups.core.pingcap.com
+- tsos.core.pingcap.com

cmd/crd-modifier/main.go

@@ -18,25 +18,32 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/parser"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
 	kyaml "sigs.k8s.io/yaml"
 )
 
 func main() {
 	var dir string
+	var chart string
 	flag.StringVar(&dir, "dir", "manifests/crd", "dir of crds")
+	flag.StringVar(&chart, "chart", "charts/tidb-operator", "dir of operator chart")
 	flag.Parse()
 
 	files, err := os.ReadDir(dir)
 	if err != nil {
 		panic(fmt.Errorf("failed to read dir %s: %w", dir, err))
 	}
 
+	var crdNames []string
+
 	for _, file := range files {
 		if file.IsDir() {
 			continue
@@ -55,6 +62,8 @@ func main() {
 			panic(err)
 		}
 
+		crdNames = append(crdNames, crd.Name)
+
 		if len(crd.Spec.Versions) == 0 {
 			fmt.Printf("  Skipping %s: no versions defined\n", fileName)
 			continue
@@ -83,6 +92,10 @@ func main() {
 			panic(err)
 		}
 	}
+
+	if err := modifyValues(chart, crdNames); err != nil {
+		panic(err)
+	}
 }
 
 // checkPathExists checks if a path of keys exists in the schema's properties.
@@ -274,3 +287,45 @@ func intersection(a, b []string) []string {
 
 	return ret
 }
+
+const (
+	defaultPerm = 0o666
+)
+
+func modifyValues(chartPath string, crds []string) error {
+	path := filepath.Join(chartPath, "values.yaml")
+
+	root, err := yaml.PathString("$")
+	if err != nil {
+		return fmt.Errorf("cannot get root: %w", err)
+	}
+
+	values, err := parser.ParseFile(path, parser.ParseComments)
+	if err != nil {
+		return fmt.Errorf("cannot parse values.yaml: %w", err)
+	}
+
+	node, err := yaml.ValueToNode(map[string][]string{
+		"crds": crds,
+	})
+	if err != nil {
+		return fmt.Errorf("cannot parse crds to ast.Node: %w", err)
+	}
+
+	if err := root.MergeFromNode(values, node); err != nil {
+		return fmt.Errorf("cannot replace node: %w", err)
+	}
+
+	data, err := io.ReadAll(values)
+	if err != nil {
+		return fmt.Errorf("cannot marshal data: %w", err)
+	}
+
+	data = append(data, '\n')
+
+	if err := os.WriteFile(path, data, defaultPerm); err != nil {
+		return fmt.Errorf("cannot write file: %w", err)
+	}
+
+	return nil
+}

cmd/tidb-operator/main.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/go-logr/logr"
 	"go.uber.org/zap/zapcore"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -42,6 +43,7 @@ import (
 
 	brv1alpha1 "github.com/pingcap/tidb-operator/api/v2/br/v1alpha1"
 	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
+	"github.com/pingcap/tidb-operator/v2/manifests"
 	"github.com/pingcap/tidb-operator/v2/pkg/adoption"
 	"github.com/pingcap/tidb-operator/v2/pkg/client"
 	"github.com/pingcap/tidb-operator/v2/pkg/controllers/br/backup"
@@ -71,6 +73,7 @@ import (
 	"github.com/pingcap/tidb-operator/v2/pkg/controllers/tiproxygroup"
 	"github.com/pingcap/tidb-operator/v2/pkg/controllers/tso"
 	"github.com/pingcap/tidb-operator/v2/pkg/controllers/tsogroup"
+	"github.com/pingcap/tidb-operator/v2/pkg/crd"
 	"github.com/pingcap/tidb-operator/v2/pkg/image"
 	"github.com/pingcap/tidb-operator/v2/pkg/metrics"
 	"github.com/pingcap/tidb-operator/v2/pkg/scheme"
@@ -84,7 +87,7 @@ import (
 	"github.com/pingcap/tidb-operator/v2/pkg/volumes"
 )
 
-var setupLog = ctrl.Log.WithName("setup").WithValues(version.Get().KeysAndValues()...)
+var setupLog = ctrl.Log.WithName("setup")
 
 type brConfig struct {
 	backupManagerImage string
@@ -98,6 +101,9 @@ func main() {
 	var probeAddr string
 	var maxConcurrentReconciles int
 	var watchDelayDuration time.Duration
+	var allowEmptyOldVersion bool
+	var skipCRDsManagement bool
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&brConf.backupManagerImage, "backup-manager-image", "", "The image of backup-manager.")
@@ -111,6 +117,9 @@ func main() {
 			"kube clients's read-after-write inconsistency(always read from cache).")
 	flag.Var(&image.PrestopChecker, "default-prestop-checker-image", "Default image of prestop checker.")
 	flag.Var(&image.ResourceSyncer, "default-resource-syncer-image", "Default image of resource syncer.")
+	flag.BoolVar(&allowEmptyOldVersion, "allow-empty-old-version", false,
+		"allow crd version is empty, if false, cannot update crd which has no version annotation")
+	flag.BoolVar(&skipCRDsManagement, "skip-crds-management", false, "if true, skip checking and applying crds")
 
 	opts := zap.Options{
 		Development:     false,
@@ -136,6 +145,10 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	ctx := logr.NewContext(context.Background(), setupLog)
+
+	setupLog.Info("start tidb operator", version.Get().KeysAndValues()...)
+
 	utilruntime.PanicHandlers = append(utilruntime.PanicHandlers, func(_ context.Context, _ any) {
 		metrics.ControllerPanic.WithLabelValues().Inc()
 	})
@@ -146,6 +159,13 @@ func main() {
 	kubeconfig.UserAgent = client.DefaultFieldManager
 	kubefeat.MustInitFeatureGates(kubeconfig)
 
+	if !skipCRDsManagement {
+		if err := applyCRDs(ctx, kubeconfig, allowEmptyOldVersion); err != nil {
+			setupLog.Error(err, "failed to apply crds")
+			os.Exit(1)
+		}
+	}
+
 	cacheOpt := cache.Options{
 		// Disable label selector for our own CRs
 		// These CRs don't need to be filtered
@@ -178,12 +198,36 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err := setup(context.Background(), mgr); err != nil {
+	if err := setup(ctx, mgr); err != nil {
 		setupLog.Error(err, "failed to setup")
 		os.Exit(1)
 	}
 }
 
+func applyCRDs(ctx context.Context, kubeconfig *rest.Config, allowEmptyOldVersion bool) error {
+	c, err := client.New(kubeconfig, ctrlcli.Options{
+		Scheme: scheme.Scheme,
+	})
+	if err != nil {
+		return fmt.Errorf("cannot new client for applying crd: %w", err)
+	}
+
+	info := version.Get()
+	cfg := crd.Config{
+		Client:               c,
+		Version:              info.GitVersion,
+		AllowEmptyOldVersion: allowEmptyOldVersion,
+		IsDirty:              info.IsDirty(),
+		CRDDir:               manifests.CRD,
+	}
+
+	if err := crd.ApplyCRDs(ctx, &cfg); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func setup(ctx context.Context, mgr ctrl.Manager) error {
 	// client of manager should be newed by options.NewClient
 	// which actually returns tidb-operator/v2/pkg/client.Client

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/go-json-experiment/json v0.0.0-20250211171154-1ae217ad3535
 	github.com/go-logr/logr v1.4.3
 	github.com/go-sql-driver/mysql v1.8.1
+	github.com/goccy/go-yaml v1.19.2
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gnostic-models v0.6.9
 	github.com/google/go-cmp v0.7.0

go.sum

@@ -243,8 +243,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
-github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
-github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
+github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=

hack/release.sh

@@ -67,7 +67,7 @@ EOF
 
 ${HELM} template tidb-operator ${OUTPUT}/tidb-operator-${RELEASE_VERSION}.tgz \
     --kube-version ${V_KUBE_VERSION} \
-    --set "operator.extraArgs={--watch-delay-duration=2s}" \
+    --set "operator.extraArgs={--watch-delay-duration=2s,--allow-empty-old-version}" \
     -n ${V_DEPLOY_NAMESPACE} \
     >> $E2E_OPERATOR
 

manifests/embed.go

@@ -0,0 +1,6 @@
+package manifests
+
+import "embed"
+
+//go:embed crd/*
+var CRD embed.FS