support join a cluster via grpc

Signed-off-by: Zhiwei Yin <[email protected]>

Author: zhiweiyin318

go.mod

@@ -40,6 +40,7 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	dario.cat/mergo v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/BurntSushi/toml v1.5.0 // indirect
@@ -53,6 +54,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/cloudevents/sdk-go/v2 v2.16.2 // indirect
 	github.com/containerd/containerd v1.7.27 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -75,6 +77,7 @@ require (
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
+cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
 dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
@@ -38,6 +40,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/cloudevents/sdk-go/v2 v2.16.2 h1:ZYDFrYke4FD+jM8TZTJJO6JhKHzOQl2oqpFK1D+NnQM=
+github.com/cloudevents/sdk-go/v2 v2.16.2/go.mod h1:laOcGImm4nVJEU+PHnUrKL56CKmRL65RlQF0kRmW/kg=
 github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII=
 github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0=
 github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4=
@@ -308,6 +312,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=

pkg/cmd/accept/exec.go

@@ -28,7 +28,9 @@ const (
 	groupNameBootstrap               = "system:bootstrappers:managedcluster"
 	userNameSignatureBootstrapPrefix = "system:bootstrap:"
 	userNameSignatureSA              = "system:serviceaccount:open-cluster-management:agent-registration-bootstrap"
+	userNameGRPCSignatureSA          = "system:serviceaccount:open-cluster-management-hub:grpc-server-sa"
 	groupNameSA                      = "system:serviceaccounts:open-cluster-management"
+	groupNameGRPC                    = "system:serviceaccounts:open-cluster-management-hub"
 	clusterLabel                     = "open-cluster-management.io/cluster-name"
 )
 
@@ -143,13 +145,15 @@ func (o *Options) approveCSR(kubeClient *kubernetes.Clientset, clusterName strin
 		for _, item := range csrs.Items {
 			// Does not have the correct name prefix
 			if !strings.HasPrefix(item.Spec.Username, userNameSignatureBootstrapPrefix) &&
-				!strings.HasPrefix(item.Spec.Username, userNameSignatureSA) {
+				!strings.HasPrefix(item.Spec.Username, userNameSignatureSA) &&
+				!strings.HasPrefix(item.Spec.Username, userNameGRPCSignatureSA) {
 				continue
 			}
 			// Check groups
 			groups := sets.NewString(item.Spec.Groups...)
 			if !groups.Has(groupNameBootstrap) &&
-				!groups.Has(groupNameSA) {
+				!groups.Has(groupNameSA) &&
+				!groups.Has(groupNameGRPC) {
 				continue
 			}
 			passedCSRs = append(passedCSRs, item)

pkg/cmd/join/cmd.go

@@ -88,5 +88,7 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
 	cmd.Flags().StringVar(&o.managedClusterArn, "managed-cluster-arn", "", "The arn of the managed cluster(i.e. EKS cluster) which will be joining the hub")
 	cmd.Flags().StringArrayVar(&o.klusterletAnnotations, "klusterlet-annotation", []string{}, fmt.Sprintf("Annotations to set on the ManagedCluster, in key=value format. Note: each key will be automatically prefixed with '%s/', if not set.", operatorv1.ClusterAnnotationsKeyPrefix))
 	cmd.Flags().StringVar(&o.klusterletValuesFile, "klusterlet-values-file", "", "The path to a YAML file containing klusterlet Helm chart values. The values from the file override both the default klusterlet chart values and the values from other flags.")
+	cmd.Flags().StringVar(&o.gRPCServer, "grpc-server", "", "The grpc server url of the hub")
+	cmd.Flags().StringVar(&o.grpcCAFile, "grpc-ca-file", "", "the file path to grpc server ca, required if --hub-grpc-server is set")
 	return cmd
 }

pkg/cmd/join/exec.go

@@ -36,8 +36,8 @@ import (
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	ocmfeature "open-cluster-management.io/api/feature"
 	operatorv1 "open-cluster-management.io/api/operator/v1"
-
 	"open-cluster-management.io/clusteradm/pkg/cmd/join/preflight"
+	"open-cluster-management.io/clusteradm/pkg/config"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
 	"open-cluster-management.io/clusteradm/pkg/helpers/klusterlet"
@@ -48,13 +48,14 @@ import (
 	"open-cluster-management.io/clusteradm/pkg/helpers/wait"
 	"open-cluster-management.io/clusteradm/pkg/version"
 	"open-cluster-management.io/ocm/pkg/operator/helpers/chart"
+	"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/cert"
+	"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc"
 	sdkhelpers "open-cluster-management.io/sdk-go/pkg/helpers"
 )
 
 const (
-	AgentNamespacePrefix = "open-cluster-management-"
-
-	OperatorNamesapce     = "open-cluster-management"
+	AgentNamespacePrefix  = "open-cluster-management-"
+	OperatorNamespace     = "open-cluster-management"
 	DefaultOperatorName   = "klusterlet"
 	AwsIrsaAuthentication = "awsirsa"
 )
@@ -191,7 +192,7 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		if err != nil {
 			return err
 		}
-		o.HubCADate = cabytes
+		o.HubCAData = cabytes
 	}
 
 	// code logic of building hub client in join process:
@@ -212,6 +213,12 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	if o.gRPCServer != "" {
+		if err = o.getGRPCCAData(externalClientUnSecure); err != nil {
+			return err
+		}
+	}
+
 	// get managed cluster externalServerURL
 	var kubeClient *kubernetes.Clientset
 	switch o.mode {
@@ -284,6 +291,11 @@ func (o *Options) validate() error {
 		return err
 	}
 
+	err = o.setGRPCConfig()
+	if err != nil {
+		return err
+	}
+
 	// get ManagedKubeconfig from given file
 	if o.mode == string(operatorv1.InstallModeHosted) {
 		managedConfig, err := os.ReadFile(o.managedKubeconfigFile)
@@ -395,7 +407,7 @@ func (o *Options) applyKlusterlet(r *reader.ResourceReader, operatorClient opera
 		o.klusterletChartConfig.NoOperator = true
 	}
 
-	crds, raw, err := chart.RenderKlusterletChart(o.klusterletChartConfig, OperatorNamesapce)
+	crds, raw, err := chart.RenderKlusterletChart(o.klusterletChartConfig, OperatorNamespace)
 	if err != nil {
 		return err
 	}
@@ -457,7 +469,7 @@ func checkIfRegistrationOperatorAvailable(f util.Factory) (bool, error) {
 		return false, err
 	}
 
-	deploy, err := client.AppsV1().Deployments(OperatorNamesapce).
+	deploy, err := client.AppsV1().Deployments(OperatorNamespace).
 		Get(context.TODO(), DefaultOperatorName, metav1.GetOptions{})
 	if err != nil {
 		if errors.IsNotFound(err) {
@@ -550,7 +562,7 @@ func waitUntilRegistrationOperatorConditionIsTrue(w io.Writer, f util.Factory, t
 
 	return helpers.WatchUntil(
 		func() (watch.Interface, error) {
-			return client.CoreV1().Pods(OperatorNamesapce).
+			return client.CoreV1().Pods(OperatorNamespace).
 				Watch(context.TODO(), metav1.ListOptions{
 					TimeoutSeconds: &timeout,
 					LabelSelector:  "app=klusterlet",
@@ -664,9 +676,9 @@ func (o *Options) createClientcmdapiv1Config(externalClientUnSecure *kubernetes.
 	bootstrapConfig := bootstrapExternalConfigUnSecure.DeepCopy()
 	bootstrapConfig.Clusters[0].Cluster.InsecureSkipTLSVerify = false
 	bootstrapConfig.Clusters[0].Cluster.Server = o.hubAPIServer
-	if o.HubCADate != nil {
+	if o.HubCAData != nil {
 		// directly set ca-data if --ca-file is set
-		bootstrapConfig.Clusters[0].Cluster.CertificateAuthorityData = o.HubCADate
+		bootstrapConfig.Clusters[0].Cluster.CertificateAuthorityData = o.HubCAData
 	} else {
 		// get ca data from externalClientUnsecure, ca may empty(cluster-info exists with no ca data)
 		ca, err := sdkhelpers.GetCACert(externalClientUnSecure)
@@ -713,6 +725,51 @@ func (o *Options) setKubeconfig() error {
 	return nil
 }
 
+func (o *Options) setGRPCConfig() error {
+	if o.gRPCServer == "" {
+		return nil
+	}
+
+	gRPCConfig := grpc.GRPCConfig{
+		CertConfig: cert.CertConfig{
+			CAData: o.GRPCCAData,
+		},
+		URL:   o.gRPCServer,
+		Token: o.token,
+	}
+
+	configStr, err := yaml.Marshal(gRPCConfig)
+	if err != nil {
+		return fmt.Errorf("failed to marshal GRPC server configuration. %v", err)
+	}
+
+	o.klusterletChartConfig.GRPCConfig = string(configStr)
+	o.klusterletChartConfig.Klusterlet.RegistrationConfiguration.RegistrationDriver = operatorv1.RegistrationDriver{
+		AuthType: operatorv1.GRPCAuthType,
+	}
+	return nil
+}
+
+func (o *Options) getGRPCCAData(kubeClient kubernetes.Interface) error {
+	if o.grpcCAFile != "" {
+		caData, err := os.ReadFile(o.grpcCAFile)
+		if err != nil {
+			return fmt.Errorf("--grpc-ca-file is invalid")
+		}
+		o.GRPCCAData = caData
+		return nil
+	}
+
+	cm, err := kubeClient.CoreV1().ConfigMaps(config.HubClusterNamespace).Get(context.TODO(),
+		config.CABundleConfigmap, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	o.GRPCCAData = []byte(cm.Data["ca-bundle.crt"])
+	return nil
+}
+
 func mergeCertificateData(caBundles ...[]byte) ([]byte, error) {
 	var all []*x509.Certificate
 	for _, caBundle := range caBundles {

pkg/cmd/join/options.go

@@ -19,8 +19,15 @@ type Options struct {
 	token string
 	// The external hub apiserver url (https://<host>:<port>)
 	hubAPIServer string
+	// The grpc server of the hub cluster
+	gRPCServer string
+
 	// The hub ca-file(optional)
 	caFile string
+
+	// The grpc ca file which can be found in the configmap ca-bundle-configmap in open-cluster-management-hub ns
+	grpcCAFile string
+
 	// the name under the cluster must be imported
 	clusterName string
 
@@ -59,11 +66,14 @@ type Options struct {
 	hubInClusterEndpoint                string
 
 	// Values below are tempoary data
-	// HubCADate: data in hub ca file
-	HubCADate []byte
+	// HubCAData: data in hub ca file
+	HubCAData []byte
 	// hub config
 	HubConfig *clientcmdapiv1.Config
 
+	// GRPCCAData: ca data used by the GRPC server
+	GRPCCAData []byte
+
 	// The URL of a forward proxy server which will be used by agnets on the managed cluster
 	// to connect to the hub cluster (optional)
 	proxyURL string

pkg/config/env.go

@@ -16,4 +16,5 @@ const (
 	ManagedClusterNamespace           = "open-cluster-management-agent"
 	ManagedProxyConfigurationName     = "cluster-proxy"
 	ImagePullSecret                   = "open-cluster-management-image-pull-credentials"
+	CABundleConfigmap                 = "ca-bundle-configmap"
 )