add resource to the transformation metrics

Author: kmala

staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -61,10 +61,10 @@ var (
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "transformation_operations_total",
-			Help:           "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption",
+			Help:           "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"transformation_type", "transformer_prefix", "status"},
+		[]string{"resource", "transformation_type", "transformer_prefix", "status"},
 	)
 
 	envelopeTransformationCacheMissTotal = metrics.NewCounter(
@@ -113,8 +113,8 @@ func RegisterMetrics() {
 
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
 // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
-func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) {
-	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, getErrorCode(err)).Inc()
+func RecordTransformation(resource, transformationType, transformerPrefix string, elapsed time.Duration, err error) {
+	transformerOperationsTotal.WithLabelValues(resource, transformationType, transformerPrefix, getErrorCode(err)).Inc()
 
 	if err == nil {
 		transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds())

staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go

@@ -27,6 +27,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/component-base/metrics/testutil"
 )
@@ -44,10 +45,11 @@ func TestTotals(t *testing.T) {
 	wrappedErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: wrappedErr}}
 
 	testCases := []struct {
-		desc    string
-		prefix  Transformer
-		metrics []string
-		want    string
+		desc      string
+		prefix    Transformer
+		metrics   []string
+		want      string
+		expectErr bool
 	}{
 		{
 			desc:   "non-status error",
@@ -56,11 +58,12 @@ func TestTotals(t *testing.T) {
 				"apiserver_storage_transformation_operations_total",
 			},
 			want: `
-				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
+				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
 				# TYPE apiserver_storage_transformation_operations_total counter
-				apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
-				apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
 				`,
+			expectErr: true,
 		},
 		{
 			desc:   "ok",
@@ -69,11 +72,12 @@ func TestTotals(t *testing.T) {
 				"apiserver_storage_transformation_operations_total",
 			},
 			want: `
-				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
+				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
 				# TYPE apiserver_storage_transformation_operations_total counter
-				apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
-				apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
 				`,
+			expectErr: false,
 		},
 		{
 			desc:   "failed precondition",
@@ -82,11 +86,12 @@ func TestTotals(t *testing.T) {
 				"apiserver_storage_transformation_operations_total",
 			},
 			want: `
-				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
+				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
 				# TYPE apiserver_storage_transformation_operations_total counter
-				apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
-				apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
 				`,
+			expectErr: true,
 		},
 		{
 			desc:   "internal",
@@ -95,11 +100,12 @@ func TestTotals(t *testing.T) {
 				"apiserver_storage_transformation_operations_total",
 			},
 			want: `
-				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
+				# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
 				# TYPE apiserver_storage_transformation_operations_total counter
-				apiserver_storage_transformation_operations_total{status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
-				apiserver_storage_transformation_operations_total{status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+				apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
 				`,
+			expectErr: true,
 		},
 		{
 			desc:   "wrapped not found error",
@@ -108,21 +114,29 @@ func TestTotals(t *testing.T) {
 				"apiserver_storage_transformation_operations_total",
 			},
 			want: `
-			# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
+			# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
 			# TYPE apiserver_storage_transformation_operations_total counter
-			apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
-			apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+			apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
+			apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
 			`,
+			expectErr: true,
 		},
 	}
 
 	RegisterMetrics()
 	transformerOperationsTotal.Reset()
+	reqCtx := request.WithRequestInfo(context.Background(), &request.RequestInfo{Resource: "test"})
 
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
-			tt.prefix.TransformToStorage(context.Background(), []byte("value"), nil)
-			tt.prefix.TransformFromStorage(context.Background(), []byte("k8s:enc:kms:v1:value"), nil)
+			_, err := tt.prefix.TransformToStorage(reqCtx, []byte("value"), nil)
+			if (err != nil) != tt.expectErr {
+				t.Fatal(err)
+			}
+			_, _, err = tt.prefix.TransformFromStorage(reqCtx, []byte("k8s:enc:kms:v1:value"), nil)
+			if (err != nil) != tt.expectErr {
+				t.Fatal(err)
+			}
 			defer transformerOperationsTotal.Reset()
 			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
 				t.Fatal(err)
@@ -229,7 +243,7 @@ func TestLatency(t *testing.T) {
 
 	for _, tt := range testCases {
 		t.Run(tt.desc, func(t *testing.T) {
-			RecordTransformation(tt.transformationType, tt.prefix, tt.elapsed, nil)
+			RecordTransformation("", tt.transformationType, tt.prefix, tt.elapsed, nil)
 			defer transformerLatencies.Reset()
 			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
 				t.Fatal(err)

staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go

@@ -105,6 +105,7 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo
 func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, bool, error) {
 	start := time.Now()
 	var errs []error
+	resource := getResourceFromContext(ctx)
 	for i, transformer := range t.transformers {
 		if bytes.HasPrefix(data, transformer.Prefix) {
 			result, stale, err := transformer.Transformer.TransformFromStorage(ctx, data[len(transformer.Prefix):], dataCtx)
@@ -116,9 +117,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
 				continue
 			}
 			if len(transformer.Prefix) == 0 {
-				RecordTransformation("from_storage", "identity", time.Since(start), err)
+				RecordTransformation(resource, "from_storage", "identity", time.Since(start), err)
 			} else {
-				RecordTransformation("from_storage", string(transformer.Prefix), time.Since(start), err)
+				RecordTransformation(resource, "from_storage", string(transformer.Prefix), time.Since(start), err)
 			}
 
 			// It is valid to have overlapping prefixes when the same encryption provider
@@ -163,16 +164,17 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
 		logTransformErr(ctx, err, "failed to decrypt data")
 		return nil, false, err
 	}
-	RecordTransformation("from_storage", "unknown", time.Since(start), t.err)
+	RecordTransformation(resource, "from_storage", "unknown", time.Since(start), t.err)
 	return nil, false, t.err
 }
 
 // TransformToStorage uses the first transformer and adds its prefix to the data.
 func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) {
 	start := time.Now()
 	transformer := t.transformers[0]
+	resource := getResourceFromContext(ctx)
 	result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx)
-	RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err)
+	RecordTransformation(resource, "to_storage", string(transformer.Prefix), time.Since(start), err)
 	if err != nil {
 		logTransformErr(ctx, err, "failed to encrypt data")
 		return nil, err
@@ -209,5 +211,11 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
 	if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
 		return reqInfo
 	}
+	klog.V(4).InfoSDepth(1, "no request info on context")
 	return &genericapirequest.RequestInfo{}
 }
+
+func getResourceFromContext(ctx context.Context) string {
+	reqInfo := getRequestInfoFromContext(ctx)
+	return schema.GroupResource{Group: reqInfo.APIGroup, Resource: reqInfo.Resource}.String()
+}