Skip to content

Commit acdd211

Browse files
zhzhuang-zjuzhzhuang-zju
committed
Adding TLS Certificate Authentication to gRPC
Signed-off-by: zhzhuang-zju <[email protected]>
1 parent 0bc96a2 commit acdd211

File tree

19 files changed

+330
-85
lines changed

19 files changed

+330
-85
lines changed

‎artifacts/deploy/karmada-descheduler.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ spec:
2727
- /bin/karmada-descheduler
2828
- --kubeconfig=/etc/kubeconfig
2929
- --bind-address=0.0.0.0
30+
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
31+
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
32+
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
3033
- --v=4
3134
livenessProbe:
3235
httpGet:
@@ -38,10 +41,16 @@ spec:
3841
periodSeconds: 15
3942
timeoutSeconds: 5
4043
volumeMounts:
44+
- name: karmada-certs
45+
mountPath: /etc/karmada/pki
46+
readOnly: true
4147
- name: kubeconfig
4248
subPath: kubeconfig
4349
mountPath: /etc/kubeconfig
4450
volumes:
51+
- name: karmada-certs
52+
secret:
53+
secretName: karmada-cert-secret
4554
- name: kubeconfig
4655
secret:
4756
secretName: kubeconfig

‎artifacts/deploy/karmada-scheduler-estimator.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ spec:
2727
- /bin/karmada-scheduler-estimator
2828
- --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig
2929
- --cluster-name={{member_cluster_name}}
30+
- --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt
31+
- --grpc-auth-key-file=/etc/karmada/pki/karmada.key
32+
- --grpc-client-ca-file=/etc/karmada/pki/ca.crt
3033
livenessProbe:
3134
httpGet:
3235
path: /healthz
@@ -37,10 +40,16 @@ spec:
3740
periodSeconds: 15
3841
timeoutSeconds: 5
3942
volumeMounts:
43+
- name: karmada-certs
44+
mountPath: /etc/karmada/pki
45+
readOnly: true
4046
- name: member-kubeconfig
4147
subPath: {{member_cluster_name}}-kubeconfig
4248
mountPath: /etc/{{member_cluster_name}}-kubeconfig
4349
volumes:
50+
- name: karmada-certs
51+
secret:
52+
secretName: karmada-cert-secret
4453
- name: member-kubeconfig
4554
secret:
4655
secretName: {{member_cluster_name}}-kubeconfig

‎artifacts/deploy/karmada-scheduler.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,21 @@ spec:
3838
- --bind-address=0.0.0.0
3939
- --secure-port=10351
4040
- --enable-scheduler-estimator=true
41+
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
42+
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
43+
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
4144
- --v=4
4245
volumeMounts:
46+
- name: karmada-certs
47+
mountPath: /etc/karmada/pki
48+
readOnly: true
4349
- name: kubeconfig
4450
subPath: kubeconfig
4551
mountPath: /etc/kubeconfig
4652
volumes:
53+
- name: karmada-certs
54+
secret:
55+
secretName: karmada-cert-secret
4756
- name: kubeconfig
4857
secret:
4958
secretName: kubeconfig

‎cmd/descheduler/app/options/options.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ type Options struct {
6262
SchedulerEstimatorServicePrefix string
6363
// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
6464
SchedulerEstimatorPort int
65+
// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
66+
SchedulerEstimatorCertFile string
67+
// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
68+
SchedulerEstimatorKeyFile string
69+
// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
70+
SchedulerEstimatorCaFile string
71+
// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
72+
InsecureSkipEstimatorVerify bool
6573
// DeschedulingInterval specifies time interval for descheduler to run.
6674
DeschedulingInterval metav1.Duration
6775
// UnschedulableThreshold specifies the period of pod unschedulable condition.
@@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
99107
fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
100108
fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
101109
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
110+
fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
111+
fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
112+
fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
113+
fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
102114
fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
103115
fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
104116
fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")

‎cmd/scheduler-estimator/app/options/options.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ type Options struct {
4040
SecurePort int
4141
// ServerPort is the port that the server gRPC serves at.
4242
ServerPort int
43+
// InsecureSkipGrpcClientVerify controls whether verifies the grpc client's certificate chain and host name.
44+
InsecureSkipGrpcClientVerify bool
45+
// GrpcAuthCertFile SSL certification file used for grpc SSL/TLS connections.
46+
GrpcAuthCertFile string
47+
// GrpcAuthKeyFile SSL key file used for grpc SSL/TLS connections.
48+
GrpcAuthKeyFile string
49+
// GrpcClientCaFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests.
50+
GrpcClientCaFile string
4351
// ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver.
4452
ClusterAPIQPS float32
4553
// ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver.
@@ -64,6 +72,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
6472
fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.")
6573
fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
6674
fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.")
75+
fs.StringVar(&o.GrpcAuthCertFile, "grpc-auth-cert-file", "", "SSL certification file used for grpc SSL/TLS connections.")
76+
fs.StringVar(&o.GrpcAuthKeyFile, "grpc-auth-key-file", "", "SSL key file used for grpc SSL/TLS connections.")
77+
fs.BoolVar(&o.InsecureSkipGrpcClientVerify, "insecure-skip-grpc-client-verify", false, "If set to true, the estimator will not verify the grpc client's certificate chain and host name. When the relevant certificates are not configured, it will not take effect.")
78+
fs.StringVar(&o.GrpcClientCaFile, "grpc-client-ca-file", "", "SSL Certificate Authority file used to verify grpc client certificates on incoming requests if --client-cert-auth flag is set.")
6779
fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.")
6880
fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.")
6981
fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.")

‎cmd/scheduler/app/options/options.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,14 @@ type Options struct {
7171
SchedulerEstimatorServicePrefix string
7272
// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
7373
SchedulerEstimatorPort int
74+
// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
75+
InsecureSkipEstimatorVerify bool
76+
// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
77+
SchedulerEstimatorCertFile string
78+
// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
79+
SchedulerEstimatorKeyFile string
80+
// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
81+
SchedulerEstimatorCaFile string
7482

7583
// EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters.
7684
EnableEmptyWorkloadPropagation bool
@@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
138146
fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
139147
fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
140148
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
149+
fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
150+
fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
151+
fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
152+
fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
141153
fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.")
142154
fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"},
143155
fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ",")))

‎cmd/scheduler/app/scheduler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
170170
scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator),
171171
scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode),
172172
scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix),
173-
scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort),
173+
scheduler.WithSchedulerEstimatorConnection(opts.SchedulerEstimatorPort, opts.SchedulerEstimatorCertFile, opts.SchedulerEstimatorKeyFile, opts.SchedulerEstimatorCaFile, opts.InsecureSkipEstimatorVerify),
174174
scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout),
175175
scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation),
176176
scheduler.WithEnableSchedulerPlugin(opts.Plugins),

‎operator/pkg/controlplane/manifests.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,9 @@ spec:
188188
- --secure-port=10351
189189
- --enable-scheduler-estimator=true
190190
- --leader-elect-resource-namespace={{ .SystemNamespace }}
191+
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
192+
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
193+
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
191194
- --v=4
192195
livenessProbe:
193196
httpGet:
@@ -199,10 +202,16 @@ spec:
199202
periodSeconds: 15
200203
timeoutSeconds: 5
201204
volumeMounts:
205+
- name: karmada-certs
206+
mountPath: /etc/karmada/pki
207+
readOnly: true
202208
- name: kubeconfig
203209
subPath: kubeconfig
204210
mountPath: /etc/karmada/kubeconfig
205211
volumes:
212+
- name: karmada-certs
213+
secret:
214+
secretName: {{ .KarmadaCertsSecret }}
206215
- name: kubeconfig
207216
secret:
208217
secretName: {{ .KubeconfigSecret }}
@@ -241,6 +250,9 @@ spec:
241250
- --kubeconfig=/etc/karmada/kubeconfig
242251
- --bind-address=0.0.0.0
243252
- --leader-elect-resource-namespace={{ .SystemNamespace }}
253+
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
254+
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
255+
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
244256
- --v=4
245257
livenessProbe:
246258
httpGet:
@@ -252,10 +264,16 @@ spec:
252264
periodSeconds: 15
253265
timeoutSeconds: 5
254266
volumeMounts:
267+
- name: karmada-certs
268+
mountPath: /etc/karmada/pki
269+
readOnly: true
255270
- name: kubeconfig
256271
subPath: kubeconfig
257272
mountPath: /etc/karmada/kubeconfig
258273
volumes:
274+
- name: karmada-certs
275+
secret:
276+
secretName: {{ .KarmadaCertsSecret }}
259277
- name: kubeconfig
260278
secret:
261279
secretName: {{ .KubeconfigSecret }}

‎pkg/descheduler/descheduler.go‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"github.com/karmada-io/karmada/pkg/util"
4646
"github.com/karmada-io/karmada/pkg/util/fedinformer"
4747
"github.com/karmada-io/karmada/pkg/util/gclient"
48+
"github.com/karmada-io/karmada/pkg/util/grpcconnection"
4849
)
4950

5051
const (
@@ -65,7 +66,7 @@ type Descheduler struct {
6566

6667
schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache
6768
schedulerEstimatorServicePrefix string
68-
schedulerEstimatorPort int
69+
schedulerEstimatorClientConfig *grpcconnection.ClientConfig
6970
schedulerEstimatorWorker util.AsyncWorker
7071

7172
unschedulableThreshold time.Duration
@@ -77,15 +78,21 @@ type Descheduler struct {
7778
func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler {
7879
factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
7980
desched := &Descheduler{
80-
KarmadaClient: karmadaClient,
81-
KubeClient: kubeClient,
82-
informerFactory: factory,
83-
bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(),
84-
bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(),
85-
clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(),
86-
clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(),
87-
schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
88-
schedulerEstimatorPort: opts.SchedulerEstimatorPort,
81+
KarmadaClient: karmadaClient,
82+
KubeClient: kubeClient,
83+
informerFactory: factory,
84+
bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(),
85+
bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(),
86+
clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(),
87+
clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(),
88+
schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
89+
schedulerEstimatorClientConfig: &grpcconnection.ClientConfig{
90+
InsecureSkipServerVerify: opts.InsecureSkipEstimatorVerify,
91+
ServerAuthCAFile: opts.SchedulerEstimatorCaFile,
92+
CertFile: opts.SchedulerEstimatorCertFile,
93+
KeyFile: opts.SchedulerEstimatorKeyFile,
94+
TargetPort: opts.SchedulerEstimatorPort,
95+
},
8996
schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix,
9097
unschedulableThreshold: opts.UnschedulableThreshold.Duration,
9198
deschedulingInterval: opts.DeschedulingInterval.Duration,
@@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() {
273280
return
274281
}
275282
for i := range clusterList.Items {
276-
if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil {
283+
if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig); err != nil {
277284
klog.Error(err)
278285
}
279286
}
@@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error {
293300
}
294301
return err
295302
}
296-
return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort)
303+
return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig)
297304
}
298305

299306
func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) {

‎pkg/estimator/client/cache.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service"
2929
"github.com/karmada-io/karmada/pkg/util"
30+
"github.com/karmada-io/karmada/pkg/util/grpcconnection"
3031
"github.com/karmada-io/karmada/pkg/util/names"
3132
)
3233

@@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim
9697
}
9798

9899
// EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator.
99-
func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error {
100+
func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcconnection.ClientConfig) error {
100101
if estimatorCache.IsEstimatorExist(name) {
101102
return nil
102103
}
103104

104105
serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem,
105-
names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port))
106+
names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.TargetPort))
106107
if err != nil {
107108
return err
108109
}
109110

110111
klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name)
111-
cc, err := util.Dial(serverAddr, 5*time.Second)
112+
cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second)
112113
if err != nil {
113114
klog.Errorf("Failed to dial cluster(%s): %v.", name, err)
114115
return err

0 commit comments

Comments
 (0)