Skip to content

Commit 429f968

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#92791 from p0lyn0mial/aggregator-dynamic-cert-reload
adds dynamic certificate reloading for kube aggregator
2 parents 7ceac2b + 25f0ebc commit 429f968

File tree

9 files changed

+438
-105
lines changed

9 files changed

+438
-105
lines changed

‎cmd/kube-apiserver/app/aggregator.go‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ package app
2121

2222
import (
2323
"fmt"
24-
"io/ioutil"
2524
"net/http"
2625
"strings"
2726
"sync"
@@ -95,28 +94,16 @@ func createAggregatorConfig(
9594
return nil, err
9695
}
9796

98-
var certBytes, keyBytes []byte
99-
if len(commandOptions.ProxyClientCertFile) > 0 && len(commandOptions.ProxyClientKeyFile) > 0 {
100-
certBytes, err = ioutil.ReadFile(commandOptions.ProxyClientCertFile)
101-
if err != nil {
102-
return nil, err
103-
}
104-
keyBytes, err = ioutil.ReadFile(commandOptions.ProxyClientKeyFile)
105-
if err != nil {
106-
return nil, err
107-
}
108-
}
109-
11097
aggregatorConfig := &aggregatorapiserver.Config{
11198
GenericConfig: &genericapiserver.RecommendedConfig{
11299
Config: genericConfig,
113100
SharedInformerFactory: externalInformers,
114101
},
115102
ExtraConfig: aggregatorapiserver.ExtraConfig{
116-
ProxyClientCert: certBytes,
117-
ProxyClientKey: keyBytes,
118-
ServiceResolver: serviceResolver,
119-
ProxyTransport: proxyTransport,
103+
ProxyClientCertFile: commandOptions.ProxyClientCertFile,
104+
ProxyClientKeyFile: commandOptions.ProxyClientKeyFile,
105+
ServiceResolver: serviceResolver,
106+
ProxyTransport: proxyTransport,
120107
},
121108
}
122109

‎staging/src/k8s.io/kube-aggregator/pkg/apiserver/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ go_test(
2222
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
2323
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
2424
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
25+
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
2526
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
2627
"//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
2728
"//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library",
@@ -62,6 +63,7 @@ go_library(
6263
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
6364
"//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
6465
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
66+
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
6567
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
6668
"//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
6769
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",

‎staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"k8s.io/client-go/pkg/version"
3131
openapicommon "k8s.io/kube-openapi/pkg/common"
3232

33+
"k8s.io/apiserver/pkg/server/dynamiccertificates"
3334
v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
3435
v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
3536
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
@@ -64,8 +65,8 @@ const legacyAPIServiceName = "v1."
6465
type ExtraConfig struct {
6566
// ProxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
6667
// this to confirm the proxy's identity
67-
ProxyClientCert []byte
68-
ProxyClientKey []byte
68+
ProxyClientCertFile string
69+
ProxyClientKeyFile string
6970

7071
// If present, the Dial method will be used for dialing out to delegate
7172
// apiservers.
@@ -108,11 +109,9 @@ type APIAggregator struct {
108109

109110
delegateHandler http.Handler
110111

111-
// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
112-
// this to confirm the proxy's identity
113-
proxyClientCert []byte
114-
proxyClientKey []byte
115-
proxyTransport *http.Transport
112+
// proxyCurrentCertKeyContent holds he client cert used to identify this proxy. Backing APIServices use this to confirm the proxy's identity
113+
proxyCurrentCertKeyContent certKeyFunc
114+
proxyTransport *http.Transport
116115

117116
// proxyHandlers are the proxy handlers that are currently registered, keyed by apiservice.name
118117
proxyHandlers map[string]*proxyHandler
@@ -178,18 +177,17 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
178177
)
179178

180179
s := &APIAggregator{
181-
GenericAPIServer: genericServer,
182-
delegateHandler: delegationTarget.UnprotectedHandler(),
183-
proxyClientCert: c.ExtraConfig.ProxyClientCert,
184-
proxyClientKey: c.ExtraConfig.ProxyClientKey,
185-
proxyTransport: c.ExtraConfig.ProxyTransport,
186-
proxyHandlers: map[string]*proxyHandler{},
187-
handledGroups: sets.String{},
188-
lister: informerFactory.Apiregistration().V1().APIServices().Lister(),
189-
APIRegistrationInformers: informerFactory,
190-
serviceResolver: c.ExtraConfig.ServiceResolver,
191-
openAPIConfig: openAPIConfig,
192-
egressSelector: c.GenericConfig.EgressSelector,
180+
GenericAPIServer: genericServer,
181+
delegateHandler: delegationTarget.UnprotectedHandler(),
182+
proxyTransport: c.ExtraConfig.ProxyTransport,
183+
proxyHandlers: map[string]*proxyHandler{},
184+
handledGroups: sets.String{},
185+
lister: informerFactory.Apiregistration().V1().APIServices().Lister(),
186+
APIRegistrationInformers: informerFactory,
187+
serviceResolver: c.ExtraConfig.ServiceResolver,
188+
openAPIConfig: openAPIConfig,
189+
egressSelector: c.GenericConfig.EgressSelector,
190+
proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
193191
}
194192

195193
apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter)
@@ -214,14 +212,30 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
214212
s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler)
215213

216214
apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().V1().APIServices(), s)
215+
if len(c.ExtraConfig.ProxyClientCertFile) > 0 && len(c.ExtraConfig.ProxyClientKeyFile) > 0 {
216+
aggregatorProxyCerts, err := dynamiccertificates.NewDynamicServingContentFromFiles("aggregator-proxy-cert", c.ExtraConfig.ProxyClientCertFile, c.ExtraConfig.ProxyClientKeyFile)
217+
if err != nil {
218+
return nil, err
219+
}
220+
if err := aggregatorProxyCerts.RunOnce(); err != nil {
221+
return nil, err
222+
}
223+
aggregatorProxyCerts.AddListener(apiserviceRegistrationController)
224+
s.proxyCurrentCertKeyContent = aggregatorProxyCerts.CurrentCertKeyContent
225+
226+
s.GenericAPIServer.AddPostStartHookOrDie("aggregator-reload-proxy-client-cert", func(context genericapiserver.PostStartHookContext) error {
227+
go aggregatorProxyCerts.Run(1, context.StopCh)
228+
return nil
229+
})
230+
}
231+
217232
availableController, err := statuscontrollers.NewAvailableConditionController(
218233
informerFactory.Apiregistration().V1().APIServices(),
219234
c.GenericConfig.SharedInformerFactory.Core().V1().Services(),
220235
c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(),
221236
apiregistrationClient.ApiregistrationV1(),
222237
c.ExtraConfig.ProxyTransport,
223-
c.ExtraConfig.ProxyClientCert,
224-
c.ExtraConfig.ProxyClientKey,
238+
(func() ([]byte, []byte))(s.proxyCurrentCertKeyContent),
225239
s.serviceResolver,
226240
c.GenericConfig.EgressSelector,
227241
)
@@ -309,12 +323,11 @@ func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
309323

310324
// register the proxy handler
311325
proxyHandler := &proxyHandler{
312-
localDelegate: s.delegateHandler,
313-
proxyClientCert: s.proxyClientCert,
314-
proxyClientKey: s.proxyClientKey,
315-
proxyTransport: s.proxyTransport,
316-
serviceResolver: s.serviceResolver,
317-
egressSelector: s.egressSelector,
326+
localDelegate: s.delegateHandler,
327+
proxyCurrentCertKeyContent: s.proxyCurrentCertKeyContent,
328+
proxyTransport: s.proxyTransport,
329+
serviceResolver: s.serviceResolver,
330+
egressSelector: s.egressSelector,
318331
}
319332
proxyHandler.updateAPIService(apiService)
320333
if s.openAPIAggregationController != nil {

‎staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiservice_controller.go‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"k8s.io/apimachinery/pkg/labels"
2525
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
2626
"k8s.io/apimachinery/pkg/util/wait"
27+
"k8s.io/apiserver/pkg/server/dynamiccertificates"
2728
"k8s.io/client-go/tools/cache"
2829
"k8s.io/client-go/util/workqueue"
2930
"k8s.io/klog/v2"
@@ -53,6 +54,8 @@ type APIServiceRegistrationController struct {
5354
queue workqueue.RateLimitingInterface
5455
}
5556

57+
var _ dynamiccertificates.Listener = &APIServiceRegistrationController{}
58+
5659
// NewAPIServiceRegistrationController returns a new APIServiceRegistrationController.
5760
func NewAPIServiceRegistrationController(apiServiceInformer informers.APIServiceInformer, apiHandlerManager APIHandlerManager) *APIServiceRegistrationController {
5861
c := &APIServiceRegistrationController{
@@ -152,7 +155,7 @@ func (c *APIServiceRegistrationController) processNextWorkItem() bool {
152155
return true
153156
}
154157

155-
func (c *APIServiceRegistrationController) enqueue(obj *v1.APIService) {
158+
func (c *APIServiceRegistrationController) enqueueInternal(obj *v1.APIService) {
156159
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
157160
if err != nil {
158161
klog.Errorf("Couldn't get key for object %#v: %v", obj, err)
@@ -165,13 +168,13 @@ func (c *APIServiceRegistrationController) enqueue(obj *v1.APIService) {
165168
func (c *APIServiceRegistrationController) addAPIService(obj interface{}) {
166169
castObj := obj.(*v1.APIService)
167170
klog.V(4).Infof("Adding %s", castObj.Name)
168-
c.enqueue(castObj)
171+
c.enqueueInternal(castObj)
169172
}
170173

171174
func (c *APIServiceRegistrationController) updateAPIService(obj, _ interface{}) {
172175
castObj := obj.(*v1.APIService)
173176
klog.V(4).Infof("Updating %s", castObj.Name)
174-
c.enqueue(castObj)
177+
c.enqueueInternal(castObj)
175178
}
176179

177180
func (c *APIServiceRegistrationController) deleteAPIService(obj interface{}) {
@@ -189,5 +192,18 @@ func (c *APIServiceRegistrationController) deleteAPIService(obj interface{}) {
189192
}
190193
}
191194
klog.V(4).Infof("Deleting %q", castObj.Name)
192-
c.enqueue(castObj)
195+
c.enqueueInternal(castObj)
196+
}
197+
198+
// Enqueue queues all apiservices to be rehandled.
199+
// This method is used by the controller to notify when the proxy cert content changes.
200+
func (c *APIServiceRegistrationController) Enqueue() {
201+
apiServices, err := c.apiServiceLister.List(labels.Everything())
202+
if err != nil {
203+
utilruntime.HandleError(err)
204+
return
205+
}
206+
for _, apiService := range apiServices {
207+
c.addAPIService(apiService)
208+
}
193209
}

‎staging/src/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,17 @@ const (
4848
aggregatedDiscoveryTimeout = 5 * time.Second
4949
)
5050

51+
type certKeyFunc func() ([]byte, []byte)
52+
5153
// proxyHandler provides a http.Handler which will proxy traffic to locations
5254
// specified by items implementing Redirector.
5355
type proxyHandler struct {
5456
// localDelegate is used to satisfy local APIServices
5557
localDelegate http.Handler
5658

57-
// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
58-
// this to confirm the proxy's identity
59-
proxyClientCert []byte
60-
proxyClientKey []byte
61-
proxyTransport *http.Transport
59+
// proxyCurrentCertKeyContent holds the client cert used to identify this proxy. Backing APIServices use this to confirm the proxy's identity
60+
proxyCurrentCertKeyContent certKeyFunc
61+
proxyTransport *http.Transport
6262

6363
// Endpoints based routing to map from cluster IP to routable IP
6464
serviceResolver ServiceResolver
@@ -248,14 +248,16 @@ func (r *proxyHandler) updateAPIService(apiService *apiregistrationv1api.APIServ
248248
return
249249
}
250250

251+
proxyClientCert, proxyClientKey := r.proxyCurrentCertKeyContent()
252+
251253
newInfo := proxyHandlingInfo{
252254
name: apiService.Name,
253255
restConfig: &restclient.Config{
254256
TLSClientConfig: restclient.TLSClientConfig{
255257
Insecure: apiService.Spec.InsecureSkipTLSVerify,
256258
ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
257-
CertData: r.proxyClientCert,
258-
KeyData: r.proxyClientKey,
259+
CertData: proxyClientCert,
260+
KeyData: proxyClientKey,
259261
CAData: apiService.Spec.CABundle,
260262
},
261263
},

0 commit comments

Comments
 (0)