Merge pull request kubernetes#123793 from aramase/aramase/f/authn_config_reload_metrics

Add metrics for authentication config reload

Author: k8s-ci-robot

pkg/controlplane/apiserver/config.go

@@ -147,7 +147,7 @@ func BuildGenericConfig(
 	ctx := wait.ContextForChannel(genericConfig.DrainedNotify())
 
 	// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
-	if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil {
+	if lastErr = s.Authentication.ApplyTo(ctx, &genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers, genericConfig.APIServerID); lastErr != nil {
 		return
 	}
 

pkg/kubeapiserver/options/authentication.go

@@ -41,6 +41,7 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	authenticationconfigmetrics "k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
 	"k8s.io/client-go/informers"
@@ -588,7 +589,16 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
 
 // ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
 // The input context controls the lifecycle of background goroutines started to reload the authentication config file.
-func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, openAPIV3Config *openapicommon.OpenAPIV3Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
+func (o *BuiltInAuthenticationOptions) ApplyTo(
+	ctx context.Context,
+	authInfo *genericapiserver.AuthenticationInfo,
+	secureServing *genericapiserver.SecureServingInfo,
+	egressSelector *egressselector.EgressSelector,
+	openAPIConfig *openapicommon.Config,
+	openAPIV3Config *openapicommon.OpenAPIV3Config,
+	extclient kubernetes.Interface,
+	versionedInformer informers.SharedInformerFactory,
+	apiServerID string) error {
 	if o == nil {
 		return nil
 	}
@@ -654,14 +664,14 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
 	authInfo.Authenticator = authenticator
 
 	if len(o.AuthenticationConfigFile) > 0 {
+		authenticationconfigmetrics.RegisterMetrics()
 		trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData
 		var mu sync.Mutex
 		go filesystem.WatchUntil(
 			ctx,
 			time.Minute,
 			o.AuthenticationConfigFile,
 			func() {
-				// TODO add metrics
 				// TODO collapse onto shared logic with DynamicEncryptionConfigContent controller
 
 				mu.Lock()
@@ -670,6 +680,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
 				authConfigBytes, err := os.ReadFile(o.AuthenticationConfigFile)
 				if err != nil {
 					klog.ErrorS(err, "failed to read authentication config file")
+					authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
 					// we do not update the tracker here because this error could eventually resolve as we keep retrying
 					return
 				}
@@ -683,13 +694,15 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
 				authConfig, err := loadAuthenticationConfigFromData(authConfigBytes)
 				if err != nil {
 					klog.ErrorS(err, "failed to load authentication config")
+					authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
 					// this config is not structurally valid and never will be, update the tracker so we stop retrying
 					trackedAuthenticationConfigData = authConfigData
 					return
 				}
 
 				if err := apiservervalidation.ValidateAuthenticationConfiguration(authConfig, authenticatorConfig.ServiceAccountIssuers).ToAggregate(); err != nil {
 					klog.ErrorS(err, "failed to validate authentication config")
+					authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
 					// this config is not semantically valid and never will be, update the tracker so we stop retrying
 					trackedAuthenticationConfigData = authConfigData
 					return
@@ -699,11 +712,14 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(ctx context.Context, authInfo *ge
 				defer timeoutCancel()
 				if err := updateAuthenticationConfig(timeoutCtx, authConfig); err != nil {
 					klog.ErrorS(err, "failed to update authentication config")
+					authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadFailure(apiServerID)
 					// we do not update the tracker here because this error could eventually resolve as we keep retrying
 					return
 				}
 
 				trackedAuthenticationConfigData = authConfigData
+				klog.InfoS("reloaded authentication config")
+				authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID)
 			},
 			func(err error) { klog.ErrorS(err, "watching authentication config file") },
 		)

staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"sync"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "authentication_config_controller"
+)
+
+var (
+	authenticationConfigAutomaticReloadsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reloads_total",
+			Help:           "Total number of automatic reloads of authentication configuration split by status and apiserver identity.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status", "apiserver_id_hash"},
+	)
+
+	authenticationConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_last_timestamp_seconds",
+			Help:           "Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status", "apiserver_id_hash"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
+		legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
+	})
+}
+
+func ResetMetricsForTest() {
+	authenticationConfigAutomaticReloadsTotal.Reset()
+	authenticationConfigAutomaticReloadLastTimestampSeconds.Reset()
+}
+
+func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	authenticationConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
+	authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
+}
+
+func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
+	authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
+}
+
+func getHash(data string) string {
+	if len(data) == 0 {
+		return ""
+	}
+	return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
+}

staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"strings"
+	"testing"
+
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+const (
+	testAPIServerID     = "testAPIServerID"
+	testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
+)
+
+func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
+	expectedValue := `
+	# HELP apiserver_authentication_config_controller_automatic_reloads_total [ALPHA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
+    # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
+    apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1
+	`
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reloads_total",
+	}
+
+	authenticationConfigAutomaticReloadsTotal.Reset()
+	RegisterMetrics()
+
+	RecordAuthenticationConfigAutomaticReloadFailure(testAPIServerID)
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
+	expectedValue := `
+	# HELP apiserver_authentication_config_controller_automatic_reloads_total [ALPHA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
+    # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
+    apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
+	`
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reloads_total",
+	}
+
+	authenticationConfigAutomaticReloadsTotal.Reset()
+	RegisterMetrics()
+
+	RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
+	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
+	testCases := []struct {
+		expectedValue string
+		resultLabel   string
+		timestamp     int64
+	}{
+		{
+			expectedValue: `
+                # HELP apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.
+                # TYPE apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds gauge
+                apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="failure"} 1.689101941e+09
+            `,
+			resultLabel: "failure",
+			timestamp:   1689101941,
+		},
+		{
+			expectedValue: `
+                # HELP apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last automatic reload of authentication configuration split by status and apiserver identity.
+                # TYPE apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds gauge
+                apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1.689101941e+09
+            `,
+			resultLabel: "success",
+			timestamp:   1689101941,
+		},
+	}
+
+	metrics := []string{
+		namespace + "_" + subsystem + "_automatic_reload_last_timestamp_seconds",
+	}
+	RegisterMetrics()
+
+	for _, tc := range testCases {
+		authenticationConfigAutomaticReloadLastTimestampSeconds.Reset()
+		authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel, testAPIServerIDHash).Set(float64(tc.timestamp))
+
+		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
+			t.Fatal(err)
+		}
+	}
+}