KEP-4633: Allow health-only anonymous auth mode.

Signed-off-by: Vinayak Goyal <[email protected]>

Author: vinayakankugoyal

cmd/kube-apiserver/app/options/options_test.go

@@ -243,9 +243,7 @@ func TestAddFlags(t *testing.T) {
 				EnableContentionProfiling: true,
 			},
 			Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-				Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
-					Allow: false,
-				},
+				Anonymous: s.Authentication.Anonymous,
 				ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
 					ClientCA: "/client-ca",
 				},
@@ -335,6 +333,6 @@ func TestAddFlags(t *testing.T) {
 	s.Authorization.AreLegacyFlagsSet = nil
 
 	if !reflect.DeepEqual(expected, s) {
-		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
+		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}, kubeoptions.AnonymousAuthenticationOptions{})))
 	}
 }

cmd/kube-controller-manager/app/options/options_test.go

@@ -30,6 +30,7 @@ import (
 	eventv1 "k8s.io/api/events/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/apis/apiserver"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	cpconfig "k8s.io/cloud-provider/config"
 	serviceconfig "k8s.io/cloud-provider/controllers/service/config"
@@ -430,6 +431,7 @@ func TestAddFlags(t *testing.T) {
 				ExtraHeaderPrefixes: []string{"x-remote-extra-"},
 			},
 			RemoteKubeConfigFileOptional: true,
+			Anonymous:                    &apiserver.AnonymousAuthConfig{Enabled: true},
 		},
 		Authorization: &apiserveroptions.DelegatingAuthorizationOptions{
 			AllowCacheTTL:                10 * time.Second,

cmd/kubelet/app/auth.go

@@ -23,6 +23,7 @@ import (
 	"reflect"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -77,7 +78,7 @@ func BuildAuthn(client authenticationclient.AuthenticationV1Interface, authn kub
 	}
 
 	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:                          authn.Anonymous.Enabled,
+		Anonymous:                          &apiserver.AnonymousAuthConfig{Enabled: authn.Anonymous.Enabled},
 		CacheTTL:                           authn.Webhook.CacheTTL.Duration,
 		ClientCertificateCAContentProvider: dynamicCAContentFromFile,
 	}

pkg/controlplane/apiserver/options/options_test.go

@@ -35,9 +35,8 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	"k8s.io/component-base/metrics"
-	netutils "k8s.io/utils/net"
-
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+	netutils "k8s.io/utils/net"
 )
 
 func TestAddFlags(t *testing.T) {
@@ -229,9 +228,7 @@ func TestAddFlags(t *testing.T) {
 			EnableContentionProfiling: true,
 		},
 		Authentication: &kubeoptions.BuiltInAuthenticationOptions{
-			Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
-				Allow: false,
-			},
+			Anonymous: s.Authentication.Anonymous,
 			ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
 				ClientCA: "/client-ca",
 			},
@@ -290,6 +287,6 @@ func TestAddFlags(t *testing.T) {
 	s.Authorization.AreLegacyFlagsSet = nil
 
 	if !reflect.DeepEqual(expected, s) {
-		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{})))
+		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}, kubeoptions.AnonymousAuthenticationOptions{})))
 	}
 }

pkg/features/kube_features.go

@@ -1203,6 +1203,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	genericfeatures.AggregatedDiscoveryEndpoint: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.33
 
+	genericfeatures.AnonymousAuthConfigurableEndpoints: {Default: false, PreRelease: featuregate.Alpha},
+
 	genericfeatures.APIListChunking: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32
 
 	genericfeatures.APIPriorityAndFairness: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31

pkg/kubeapiserver/authenticator/config.go

@@ -55,7 +55,10 @@ import (
 
 // Config contains the data on how to authenticate a request to the Kube API Server
 type Config struct {
-	Anonymous      bool
+	// Anonymous holds the effective anonymous config, specified either via config file
+	// (hoisted out of AuthenticationConfig) or via flags (constructed from flag-specified values).
+	Anonymous apiserver.AnonymousAuthConfig
+
 	BootstrapToken bool
 
 	TokenAuthFile               string
@@ -212,8 +215,8 @@ func (config Config) New(serverLifecycle context.Context) (authenticator.Request
 	}
 
 	if len(authenticators) == 0 {
-		if config.Anonymous {
-			return anonymous.NewAuthenticator(), nil, &securityDefinitionsV2, securitySchemesV3, nil
+		if config.Anonymous.Enabled {
+			return anonymous.NewAuthenticator(config.Anonymous.Conditions), nil, &securityDefinitionsV2, securitySchemesV3, nil
 		}
 		return nil, nil, &securityDefinitionsV2, securitySchemesV3, nil
 	}
@@ -222,10 +225,10 @@ func (config Config) New(serverLifecycle context.Context) (authenticator.Request
 
 	authenticator = group.NewAuthenticatedGroupAdder(authenticator)
 
-	if config.Anonymous {
+	if config.Anonymous.Enabled {
 		// If the authenticator chain returns an error, return an error (don't consider a bad bearer token
 		// or invalid username/password combination anonymous).
-		authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
+		authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator(config.Anonymous.Conditions))
 	}
 
 	return authenticator, updateAuthenticationConfig, &securityDefinitionsV2, securitySchemesV3, nil

pkg/kubeapiserver/options/authentication.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"reflect"
 	"strings"
 	"sync"
 	"time"
@@ -32,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/apis/apiserver/install"
@@ -95,7 +97,8 @@ type BuiltInAuthenticationOptions struct {
 
 // AnonymousAuthenticationOptions contains anonymous authentication options for API Server
 type AnonymousAuthenticationOptions struct {
-	Allow bool
+	Allow       bool
+	areFlagsSet func() bool
 }
 
 // BootstrapTokenAuthenticationOptions contains bootstrap token authentication options for API Server
@@ -169,7 +172,10 @@ func (o *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
 
 // WithAnonymous set default value for anonymous authentication
 func (o *BuiltInAuthenticationOptions) WithAnonymous() *BuiltInAuthenticationOptions {
-	o.Anonymous = &AnonymousAuthenticationOptions{Allow: true}
+	o.Anonymous = &AnonymousAuthenticationOptions{
+		Allow:       true,
+		areFlagsSet: func() bool { return false },
+	}
 	return o
 }
 
@@ -294,6 +300,14 @@ func (o *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 		return
 	}
 
+	fs.StringVar(&o.AuthenticationConfigFile, "authentication-config", o.AuthenticationConfigFile, ""+
+		"File with Authentication Configuration to configure the JWT Token authenticator or the anonymous authenticator. "+
+		"Note: This feature is in Alpha since v1.29."+
+		"--feature-gate=StructuredAuthenticationConfiguration=true needs to be set for enabling this feature."+
+		"This feature is mutually exclusive with the oidc-* flags."+
+		"To configure anonymous authenticator you need to enable --feature-gate=AnonymousAuthConfigurableEndpoints."+
+		"When you configure anonymous authenticator in the authentication config you cannot use the --anonymous-auth flag.")
+
 	fs.StringSliceVar(&o.APIAudiences, "api-audiences", o.APIAudiences, ""+
 		"Identifiers of the API. The service account token authenticator will validate that "+
 		"tokens used against the API are bound to at least one of these audiences. If the "+
@@ -305,6 +319,10 @@ func (o *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 			"Enables anonymous requests to the secure port of the API server. "+
 			"Requests that are not rejected by another authentication method are treated as anonymous requests. "+
 			"Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated.")
+
+		o.Anonymous.areFlagsSet = func() bool {
+			return fs.Changed("anonymous-auth")
+		}
 	}
 
 	if o.BootstrapToken != nil {
@@ -358,12 +376,6 @@ func (o *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 			"If set, the claim is verified to be present in the ID Token with a matching value. "+
 			"Repeat this flag to specify multiple claims.")
 
-		fs.StringVar(&o.AuthenticationConfigFile, "authentication-config", o.AuthenticationConfigFile, ""+
-			"File with Authentication Configuration to configure the JWT Token authenticator. "+
-			"Note: This feature is in Alpha since v1.29."+
-			"--feature-gate=StructuredAuthenticationConfiguration=true needs to be set for enabling this feature."+
-			"This feature is mutually exclusive with the oidc-* flags.")
-
 		o.OIDC.areFlagsConfigured = func() bool {
 			return fs.Changed(oidcIssuerURLFlag) ||
 				fs.Changed(oidcClientIDFlag) ||
@@ -452,10 +464,6 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
 		TokenFailureCacheTTL: o.TokenFailureCacheTTL,
 	}
 
-	if o.Anonymous != nil {
-		ret.Anonymous = o.Anonymous.Allow
-	}
-
 	if o.BootstrapToken != nil {
 		ret.BootstrapToken = o.BootstrapToken.Enable
 	}
@@ -469,12 +477,18 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
 	}
 
 	// When the StructuredAuthenticationConfiguration feature is enabled and the authentication config file is provided,
-	// load the authentication config from the file.
+	// load the authentication config from the file, otherwise set up an empty configuration.
 	if len(o.AuthenticationConfigFile) > 0 {
 		var err error
 		if ret.AuthenticationConfig, ret.AuthenticationConfigData, err = loadAuthenticationConfig(o.AuthenticationConfigFile); err != nil {
 			return kubeauthenticator.Config{}, err
 		}
+	} else {
+		ret.AuthenticationConfig = &apiserver.AuthenticationConfiguration{}
+	}
+
+	// Set up JWT authenticators from config file or from flags
+	if len(o.AuthenticationConfigFile) > 0 {
 		// all known signing algs are allowed when using authentication config
 		// TODO: what we really want to express is 'any alg is fine as long it matches a public key'
 		ret.OIDCSigningAlgs = oidc.AllValidSigningAlgorithms()
@@ -532,20 +546,30 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
 			jwtAuthenticator.ClaimValidationRules = claimValidationRules
 		}
 
-		authConfig := &apiserver.AuthenticationConfiguration{
-			JWT: []apiserver.JWTAuthenticator{jwtAuthenticator},
-		}
+		ret.AuthenticationConfig.JWT = []apiserver.JWTAuthenticator{jwtAuthenticator}
 
-		ret.AuthenticationConfig = authConfig
 		ret.OIDCSigningAlgs = o.OIDC.SigningAlgs
 	}
 
-	if ret.AuthenticationConfig != nil {
-		if err := apiservervalidation.ValidateAuthenticationConfiguration(ret.AuthenticationConfig, ret.ServiceAccountIssuers).ToAggregate(); err != nil {
-			return kubeauthenticator.Config{}, err
+	// Set up anonymous authenticator from config file or flags
+	if o.Anonymous != nil {
+		switch {
+		case ret.AuthenticationConfig.Anonymous != nil && o.Anonymous.areFlagsSet():
+			// Flags and config file are mutually exclusive
+			return kubeauthenticator.Config{}, field.Forbidden(field.NewPath("anonymous"), "--anonynous-auth flag cannot be set when anonymous field is configured in authentication configuration file")
+		case ret.AuthenticationConfig.Anonymous != nil:
+			// Use the config-file-specified values
+			ret.Anonymous = *ret.AuthenticationConfig.Anonymous
+		default:
+			// Use the flag-specified values
+			ret.Anonymous = apiserver.AnonymousAuthConfig{Enabled: o.Anonymous.Allow}
 		}
 	}
 
+	if err := apiservervalidation.ValidateAuthenticationConfiguration(ret.AuthenticationConfig, ret.ServiceAccountIssuers).ToAggregate(); err != nil {
+		return kubeauthenticator.Config{}, err
+	}
+
 	if o.RequestHeader != nil {
 		var err error
 		ret.RequestHeaderConfig, err = o.RequestHeader.ToAuthenticationRequestHeaderConfig()
@@ -667,6 +691,10 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
 		authenticationconfigmetrics.RegisterMetrics()
 		trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData
 		var mu sync.Mutex
+
+		// ensure anonymous config doesn't change on reload
+		originalFileAnonymousConfig := authenticatorConfig.AuthenticationConfig.DeepCopy().Anonymous
+
 		go filesystem.WatchUntil(
 			ctx,
 			time.Minute,
@@ -700,7 +728,11 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
 					return
 				}
 
-				if err := apiservervalidation.ValidateAuthenticationConfiguration(authConfig, authenticatorConfig.ServiceAccountIssuers).ToAggregate(); err != nil {
+				validationErrs := apiservervalidation.ValidateAuthenticationConfiguration(authConfig, authenticatorConfig.ServiceAccountIssuers)
+				if !reflect.DeepEqual(originalFileAnonymousConfig, authConfig.Anonymous) {
+					validationErrs = append(validationErrs, field.Forbidden(field.NewPath("anonymous"), "changed from initial configuration file"))
+				}
+				if err := validationErrs.ToAggregate(); err != nil {
 					klog.ErrorS(err, "failed to validate authentication config")
 					authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
 					// this config is not semantically valid and never will be, update the tracker so we stop retrying