Skip to content

Commit 77737c3

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#126512 from kmala/metrics
add resource to the transformation metrics
2 parents 44ce613 + 3a8df1e commit 77737c3

File tree

5 files changed

+134
-70
lines changed

5 files changed

+134
-70
lines changed

‎staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,10 @@ var (
6161
Namespace: namespace,
6262
Subsystem: subsystem,
6363
Name: "transformation_operations_total",
64-
Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption",
64+
Help: "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.",
6565
StabilityLevel: metrics.ALPHA,
6666
},
67-
[]string{"transformation_type", "transformer_prefix", "status"},
67+
[]string{"resource", "transformation_type", "transformer_prefix", "status"},
6868
)
6969

7070
envelopeTransformationCacheMissTotal = metrics.NewCounter(
@@ -113,8 +113,8 @@ func RegisterMetrics() {
113113

114114
// RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
115115
// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
116-
func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) {
117-
transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, getErrorCode(err)).Inc()
116+
func RecordTransformation(resource, transformationType, transformerPrefix string, elapsed time.Duration, err error) {
117+
transformerOperationsTotal.WithLabelValues(resource, transformationType, transformerPrefix, getErrorCode(err)).Inc()
118118

119119
if err == nil {
120120
transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds())

‎staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go‎

Lines changed: 36 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"google.golang.org/grpc/codes"
2828
"google.golang.org/grpc/status"
2929

30+
"k8s.io/apiserver/pkg/endpoints/request"
3031
"k8s.io/component-base/metrics/legacyregistry"
3132
"k8s.io/component-base/metrics/testutil"
3233
)
@@ -44,10 +45,11 @@ func TestTotals(t *testing.T) {
4445
wrappedErrTransformer := PrefixTransformer{Prefix: []byte("k8s:enc:kms:v1:"), Transformer: &testTransformer{err: wrappedErr}}
4546

4647
testCases := []struct {
47-
desc string
48-
prefix Transformer
49-
metrics []string
50-
want string
48+
desc string
49+
prefix Transformer
50+
metrics []string
51+
want string
52+
expectErr bool
5153
}{
5254
{
5355
desc: "non-status error",
@@ -56,11 +58,12 @@ func TestTotals(t *testing.T) {
5658
"apiserver_storage_transformation_operations_total",
5759
},
5860
want: `
59-
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
61+
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
6062
# TYPE apiserver_storage_transformation_operations_total counter
61-
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
62-
apiserver_storage_transformation_operations_total{status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
63+
apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
64+
apiserver_storage_transformation_operations_total{resource="test",status="unknown-non-grpc",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
6365
`,
66+
expectErr: true,
6467
},
6568
{
6669
desc: "ok",
@@ -69,11 +72,12 @@ func TestTotals(t *testing.T) {
6972
"apiserver_storage_transformation_operations_total",
7073
},
7174
want: `
72-
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
75+
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
7376
# TYPE apiserver_storage_transformation_operations_total counter
74-
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
75-
apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
77+
apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
78+
apiserver_storage_transformation_operations_total{resource="test",status="OK",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
7679
`,
80+
expectErr: false,
7781
},
7882
{
7983
desc: "failed precondition",
@@ -82,11 +86,12 @@ func TestTotals(t *testing.T) {
8286
"apiserver_storage_transformation_operations_total",
8387
},
8488
want: `
85-
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
89+
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
8690
# TYPE apiserver_storage_transformation_operations_total counter
87-
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
88-
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
91+
apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
92+
apiserver_storage_transformation_operations_total{resource="test",status="FailedPrecondition",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
8993
`,
94+
expectErr: true,
9095
},
9196
{
9297
desc: "internal",
@@ -95,11 +100,12 @@ func TestTotals(t *testing.T) {
95100
"apiserver_storage_transformation_operations_total",
96101
},
97102
want: `
98-
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
103+
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
99104
# TYPE apiserver_storage_transformation_operations_total counter
100-
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
101-
apiserver_storage_transformation_operations_total{status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
105+
apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
106+
apiserver_storage_transformation_operations_total{resource="test",status="Internal",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
102107
`,
108+
expectErr: true,
103109
},
104110
{
105111
desc: "wrapped not found error",
@@ -108,21 +114,29 @@ func TestTotals(t *testing.T) {
108114
"apiserver_storage_transformation_operations_total",
109115
},
110116
want: `
111-
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption
117+
# HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. The status, resource, and transformation_type fields can be used for alerting purposes. For example, you can monitor for encryption/decryption failures using the transformation_type (e.g., from_storage for decryption and to_storage for encryption). Additionally, these fields can be used to ensure that the correct transformers are applied to each resource.
112118
# TYPE apiserver_storage_transformation_operations_total counter
113-
apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
114-
apiserver_storage_transformation_operations_total{status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
119+
apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="from_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
120+
apiserver_storage_transformation_operations_total{resource="test",status="NotFound",transformation_type="to_storage",transformer_prefix="k8s:enc:kms:v1:"} 1
115121
`,
122+
expectErr: true,
116123
},
117124
}
118125

119126
RegisterMetrics()
120127
transformerOperationsTotal.Reset()
128+
reqCtx := request.WithRequestInfo(context.Background(), &request.RequestInfo{Resource: "test"})
121129

122130
for _, tt := range testCases {
123131
t.Run(tt.desc, func(t *testing.T) {
124-
tt.prefix.TransformToStorage(context.Background(), []byte("value"), nil)
125-
tt.prefix.TransformFromStorage(context.Background(), []byte("k8s:enc:kms:v1:value"), nil)
132+
_, err := tt.prefix.TransformToStorage(reqCtx, []byte("value"), nil)
133+
if (err != nil) != tt.expectErr {
134+
t.Fatal(err)
135+
}
136+
_, _, err = tt.prefix.TransformFromStorage(reqCtx, []byte("k8s:enc:kms:v1:value"), nil)
137+
if (err != nil) != tt.expectErr {
138+
t.Fatal(err)
139+
}
126140
defer transformerOperationsTotal.Reset()
127141
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
128142
t.Fatal(err)
@@ -229,7 +243,7 @@ func TestLatency(t *testing.T) {
229243

230244
for _, tt := range testCases {
231245
t.Run(tt.desc, func(t *testing.T) {
232-
RecordTransformation(tt.transformationType, tt.prefix, tt.elapsed, nil)
246+
RecordTransformation("", tt.transformationType, tt.prefix, tt.elapsed, nil)
233247
defer transformerLatencies.Reset()
234248
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
235249
t.Fatal(err)

‎staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo
105105
func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, bool, error) {
106106
start := time.Now()
107107
var errs []error
108+
resource := getResourceFromContext(ctx)
108109
for i, transformer := range t.transformers {
109110
if bytes.HasPrefix(data, transformer.Prefix) {
110111
result, stale, err := transformer.Transformer.TransformFromStorage(ctx, data[len(transformer.Prefix):], dataCtx)
@@ -116,9 +117,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
116117
continue
117118
}
118119
if len(transformer.Prefix) == 0 {
119-
RecordTransformation("from_storage", "identity", time.Since(start), err)
120+
RecordTransformation(resource, "from_storage", "identity", time.Since(start), err)
120121
} else {
121-
RecordTransformation("from_storage", string(transformer.Prefix), time.Since(start), err)
122+
RecordTransformation(resource, "from_storage", string(transformer.Prefix), time.Since(start), err)
122123
}
123124

124125
// It is valid to have overlapping prefixes when the same encryption provider
@@ -163,16 +164,17 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
163164
logTransformErr(ctx, err, "failed to decrypt data")
164165
return nil, false, err
165166
}
166-
RecordTransformation("from_storage", "unknown", time.Since(start), t.err)
167+
RecordTransformation(resource, "from_storage", "unknown", time.Since(start), t.err)
167168
return nil, false, t.err
168169
}
169170

170171
// TransformToStorage uses the first transformer and adds its prefix to the data.
171172
func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx Context) ([]byte, error) {
172173
start := time.Now()
173174
transformer := t.transformers[0]
175+
resource := getResourceFromContext(ctx)
174176
result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx)
175-
RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err)
177+
RecordTransformation(resource, "to_storage", string(transformer.Prefix), time.Since(start), err)
176178
if err != nil {
177179
logTransformErr(ctx, err, "failed to encrypt data")
178180
return nil, err
@@ -209,5 +211,11 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
209211
if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
210212
return reqInfo
211213
}
214+
klog.V(4).InfoSDepth(1, "no request info on context")
212215
return &genericapirequest.RequestInfo{}
213216
}
217+
218+
func getResourceFromContext(ctx context.Context) string {
219+
reqInfo := getRequestInfoFromContext(ctx)
220+
return schema.GroupResource{Group: reqInfo.APIGroup, Resource: reqInfo.Resource}.String()
221+
}

0 commit comments

Comments
 (0)