Skip to content

Commit 77ecfb7

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#123525 from enj/enj/f/authn_config_reload
Add dynamic reload support for authentication configuration
2 parents eafd289 + b4935d9 commit 77ecfb7

File tree

9 files changed

+855
-105
lines changed

9 files changed

+855
-105
lines changed

‎pkg/controlplane/apiserver/config.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,16 @@ func BuildGenericConfig(
144144
return
145145
}
146146

147+
ctx := wait.ContextForChannel(genericConfig.DrainedNotify())
148+
147149
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
148-
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
150+
if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
149151
return
150152
}
151153

152154
var enablesRBAC bool
153155
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, enablesRBAC, err = BuildAuthorizer(
154-
wait.ContextForChannel(genericConfig.ShutdownInitiatedNotify()),
156+
ctx,
155157
s,
156158
genericConfig.EgressSelector,
157159
genericConfig.APIServerID,

‎pkg/kubeapiserver/authenticator/config.go‎

Lines changed: 121 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@ limitations under the License.
1717
package authenticator
1818

1919
import (
20+
"context"
2021
"errors"
2122
"fmt"
23+
"sync/atomic"
2224
"time"
2325

26+
utilerrors "k8s.io/apimachinery/pkg/util/errors"
2427
utilnet "k8s.io/apimachinery/pkg/util/net"
2528
"k8s.io/apimachinery/pkg/util/wait"
2629
"k8s.io/apiserver/pkg/apis/apiserver"
@@ -57,6 +60,7 @@ type Config struct {
5760

5861
TokenAuthFile string
5962
AuthenticationConfig *apiserver.AuthenticationConfiguration
63+
AuthenticationConfigData string
6064
OIDCSigningAlgs []string
6165
ServiceAccountKeyFiles []string
6266
ServiceAccountLookup bool
@@ -90,7 +94,7 @@ type Config struct {
9094

9195
// New returns an authenticator.Request or an error that supports the standard
9296
// Kubernetes authentication mechanisms.
93-
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, spec3.SecuritySchemes, error) {
97+
func (config Config) New(serverLifecycle context.Context) (authenticator.Request, func(context.Context, *apiserver.AuthenticationConfiguration) error, *spec.SecurityDefinitions, spec3.SecuritySchemes, error) {
9498
var authenticators []authenticator.Request
9599
var tokenAuthenticators []authenticator.Token
96100
securityDefinitionsV2 := spec.SecurityDefinitions{}
@@ -119,21 +123,21 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
119123
if len(config.TokenAuthFile) > 0 {
120124
tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
121125
if err != nil {
122-
return nil, nil, nil, err
126+
return nil, nil, nil, nil, err
123127
}
124128
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
125129
}
126130
if len(config.ServiceAccountKeyFiles) > 0 {
127131
serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter, config.SecretsWriter)
128132
if err != nil {
129-
return nil, nil, nil, err
133+
return nil, nil, nil, nil, err
130134
}
131135
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
132136
}
133137
if len(config.ServiceAccountIssuers) > 0 {
134138
serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
135139
if err != nil {
136-
return nil, nil, nil, err
140+
return nil, nil, nil, nil, err
137141
}
138142
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
139143
}
@@ -148,33 +152,33 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
148152
// cache misses for all requests using the other. While the service account plugin
149153
// simply returns an error, the OpenID Connect plugin may query the provider to
150154
// update the keys, causing performance hits.
155+
var updateAuthenticationConfig func(context.Context, *apiserver.AuthenticationConfiguration) error
151156
if config.AuthenticationConfig != nil {
152-
for _, jwtAuthenticator := range config.AuthenticationConfig.JWT {
153-
var oidcCAContent oidc.CAContentProvider
154-
if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 {
155-
var oidcCAError error
156-
oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent("oidc-authenticator", []byte(jwtAuthenticator.Issuer.CertificateAuthority))
157-
if oidcCAError != nil {
158-
return nil, nil, nil, oidcCAError
159-
}
160-
}
161-
oidcAuth, err := oidc.New(oidc.Options{
162-
JWTAuthenticator: jwtAuthenticator,
163-
CAContentProvider: oidcCAContent,
164-
SupportedSigningAlgs: config.OIDCSigningAlgs,
165-
DisallowedIssuers: config.ServiceAccountIssuers,
166-
})
167-
if err != nil {
168-
return nil, nil, nil, err
169-
}
170-
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, oidcAuth))
157+
initialJWTAuthenticator, err := newJWTAuthenticator(serverLifecycle, config.AuthenticationConfig, config.OIDCSigningAlgs, config.APIAudiences, config.ServiceAccountIssuers)
158+
if err != nil {
159+
return nil, nil, nil, nil, err
171160
}
161+
162+
jwtAuthenticatorPtr := &atomic.Pointer[jwtAuthenticatorWithCancel]{}
163+
jwtAuthenticatorPtr.Store(initialJWTAuthenticator)
164+
165+
updateAuthenticationConfig = (&authenticationConfigUpdater{
166+
serverLifecycle: serverLifecycle,
167+
config: config,
168+
jwtAuthenticatorPtr: jwtAuthenticatorPtr,
169+
}).updateAuthenticationConfig
170+
171+
tokenAuthenticators = append(tokenAuthenticators,
172+
authenticator.TokenFunc(func(ctx context.Context, token string) (*authenticator.Response, bool, error) {
173+
return jwtAuthenticatorPtr.Load().jwtAuthenticator.AuthenticateToken(ctx, token)
174+
}),
175+
)
172176
}
173177

174178
if len(config.WebhookTokenAuthnConfigFile) > 0 {
175179
webhookTokenAuth, err := newWebhookTokenAuthenticator(config)
176180
if err != nil {
177-
return nil, nil, nil, err
181+
return nil, nil, nil, nil, err
178182
}
179183

180184
tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
@@ -209,9 +213,9 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
209213

210214
if len(authenticators) == 0 {
211215
if config.Anonymous {
212-
return anonymous.NewAuthenticator(), &securityDefinitionsV2, securitySchemesV3, nil
216+
return anonymous.NewAuthenticator(), nil, &securityDefinitionsV2, securitySchemesV3, nil
213217
}
214-
return nil, &securityDefinitionsV2, securitySchemesV3, nil
218+
return nil, nil, &securityDefinitionsV2, securitySchemesV3, nil
215219
}
216220

217221
authenticator := union.New(authenticators...)
@@ -224,7 +228,97 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, sp
224228
authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
225229
}
226230

227-
return authenticator, &securityDefinitionsV2, securitySchemesV3, nil
231+
return authenticator, updateAuthenticationConfig, &securityDefinitionsV2, securitySchemesV3, nil
232+
}
233+
234+
type jwtAuthenticatorWithCancel struct {
235+
jwtAuthenticator authenticator.Token
236+
healthCheck func() error
237+
cancel func()
238+
}
239+
240+
func newJWTAuthenticator(serverLifecycle context.Context, config *apiserver.AuthenticationConfiguration, oidcSigningAlgs []string, apiAudiences authenticator.Audiences, disallowedIssuers []string) (_ *jwtAuthenticatorWithCancel, buildErr error) {
241+
ctx, cancel := context.WithCancel(serverLifecycle)
242+
243+
defer func() {
244+
if buildErr != nil {
245+
cancel()
246+
}
247+
}()
248+
var jwtAuthenticators []authenticator.Token
249+
var healthChecks []func() error
250+
for _, jwtAuthenticator := range config.JWT {
251+
// TODO remove this CAContentProvider indirection
252+
var oidcCAContent oidc.CAContentProvider
253+
if len(jwtAuthenticator.Issuer.CertificateAuthority) > 0 {
254+
var oidcCAError error
255+
oidcCAContent, oidcCAError = dynamiccertificates.NewStaticCAContent("oidc-authenticator", []byte(jwtAuthenticator.Issuer.CertificateAuthority))
256+
if oidcCAError != nil {
257+
return nil, oidcCAError
258+
}
259+
}
260+
oidcAuth, err := oidc.New(ctx, oidc.Options{
261+
JWTAuthenticator: jwtAuthenticator,
262+
CAContentProvider: oidcCAContent,
263+
SupportedSigningAlgs: oidcSigningAlgs,
264+
DisallowedIssuers: disallowedIssuers,
265+
})
266+
if err != nil {
267+
return nil, err
268+
}
269+
jwtAuthenticators = append(jwtAuthenticators, oidcAuth)
270+
healthChecks = append(healthChecks, oidcAuth.HealthCheck)
271+
}
272+
return &jwtAuthenticatorWithCancel{
273+
jwtAuthenticator: authenticator.WrapAudienceAgnosticToken(apiAudiences, tokenunion.NewFailOnError(jwtAuthenticators...)), // this handles the empty jwtAuthenticators slice case correctly
274+
healthCheck: func() error {
275+
var errs []error
276+
for _, check := range healthChecks {
277+
if err := check(); err != nil {
278+
errs = append(errs, err)
279+
}
280+
}
281+
return utilerrors.NewAggregate(errs)
282+
},
283+
cancel: cancel,
284+
}, nil
285+
}
286+
287+
type authenticationConfigUpdater struct {
288+
serverLifecycle context.Context
289+
config Config
290+
jwtAuthenticatorPtr *atomic.Pointer[jwtAuthenticatorWithCancel]
291+
}
292+
293+
// the input ctx controls the timeout for updateAuthenticationConfig to return, not the lifetime of the constructed authenticators.
294+
func (c *authenticationConfigUpdater) updateAuthenticationConfig(ctx context.Context, authConfig *apiserver.AuthenticationConfiguration) error {
295+
updatedJWTAuthenticator, err := newJWTAuthenticator(c.serverLifecycle, authConfig, c.config.OIDCSigningAlgs, c.config.APIAudiences, c.config.ServiceAccountIssuers)
296+
if err != nil {
297+
return err
298+
}
299+
300+
var lastErr error
301+
if waitErr := wait.PollUntilContextCancel(ctx, 10*time.Second, true, func(_ context.Context) (done bool, err error) {
302+
lastErr = updatedJWTAuthenticator.healthCheck()
303+
return lastErr == nil, nil
304+
}); lastErr != nil || waitErr != nil {
305+
updatedJWTAuthenticator.cancel()
306+
return utilerrors.NewAggregate([]error{lastErr, waitErr}) // filters out nil errors
307+
}
308+
309+
oldJWTAuthenticator := c.jwtAuthenticatorPtr.Swap(updatedJWTAuthenticator)
310+
go func() {
311+
t := time.NewTimer(time.Minute)
312+
defer t.Stop()
313+
select {
314+
case <-c.serverLifecycle.Done():
315+
case <-t.C:
316+
}
317+
// TODO maybe track requests so we know when this is safe to do
318+
oldJWTAuthenticator.cancel()
319+
}()
320+
321+
return nil
228322
}
229323

230324
// IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file

0 commit comments

Comments
 (0)