From 110f7410bfa36817afef31ad00756f206e62a12f Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Fri, 22 Nov 2024 14:29:10 -0800 Subject: [PATCH 1/3] Support prometheus metrics Following metrics added: certmanager_csi_certificate_request_expiration_timestamp_seconds certmanager_csi_certificate_request_ready_status certmanager_csi_certificate_request_renewal_timestamp_seconds certmanager_csi_driver_issue_call_count certmanager_csi_driver_issue_error_count certmanager_csi_managed_certificate_count certmanager_csi_managed_volume_count fixes: #60 Signed-off-by: Jing Liu --- go.mod | 3 +- manager/manager.go | 53 ++++- manager/manager_test.go | 17 +- metrics/certificaterequest.go | 102 ++++++++ metrics/certificaterequest_test.go | 358 +++++++++++++++++++++++++++++ metrics/metrics.go | 192 ++++++++++++++++ test/driver/driver_testing.go | 3 + test/integration/metrics_test.go | 235 +++++++++++++++++++ test/util/testutil.go | 6 +- 9 files changed, 951 insertions(+), 18 deletions(-) create mode 100644 metrics/certificaterequest.go create mode 100644 metrics/certificaterequest_test.go create mode 100644 metrics/metrics.go create mode 100644 test/integration/metrics_test.go diff --git a/go.mod b/go.mod index d1c8d76..f96bf21 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/container-storage-interface/spec v1.10.0 github.com/go-logr/logr v1.4.2 github.com/kubernetes-csi/csi-lib-utils v0.19.0 + github.com/prometheus/client_golang v1.20.4 github.com/stretchr/testify v1.9.0 google.golang.org/grpc v1.66.2 k8s.io/apimachinery v0.31.1 @@ -39,6 +40,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -48,7 +50,6 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/manager/manager.go b/manager/manager.go index 869693d..c8c583e 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -47,6 +47,7 @@ import ( internalapi "github.com/cert-manager/csi-lib/internal/api" internalapiutil "github.com/cert-manager/csi-lib/internal/api/util" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -89,6 +90,9 @@ type Options struct { // RenewalBackoffConfig configures the exponential backoff applied to certificate renewal failures. RenewalBackoffConfig *wait.Backoff + + // Metrics is used for exposing Prometheus metrics + Metrics *metrics.Metrics } // NewManager constructs a new manager used to manage volumes containing @@ -126,6 +130,9 @@ func NewManager(opts Options) (*Manager, error) { if opts.Log == nil { return nil, errors.New("log must be set") } + if opts.Metrics == nil { + opts.Metrics = metrics.New(opts.Log) + } if opts.MetadataReader == nil { return nil, errors.New("MetadataReader must be set") } @@ -241,6 +248,7 @@ func NewManager(opts Options) (*Manager, error) { metadataReader: opts.MetadataReader, clock: opts.Clock, log: *opts.Log, + metrics: opts.Metrics, generatePrivateKey: opts.GeneratePrivateKey, generateRequest: opts.GenerateRequest, @@ -375,6 +383,9 @@ type Manager struct { // No thread safety is added around this field, and it MUST NOT be used for any implementation logic. // It should not be used full-stop :). doNotUse_CallOnEachIssue func() + + // metrics is used to expose Prometheus + metrics *metrics.Metrics } // issue will step through the entire issuance flow for a volume. @@ -387,6 +398,9 @@ func (m *Manager) issue(ctx context.Context, volumeID string) error { log := m.log.WithValues("volume_id", volumeID) log.Info("Processing issuance") + // Increase issue count + m.metrics.IncrementIssueCallCount(m.nodeNameHash, volumeID) + if err := m.cleanupStaleRequests(ctx, log, volumeID); err != nil { return fmt.Errorf("cleaning up stale requests: %w", err) } @@ -594,7 +608,7 @@ func (m *Manager) handleRequest(ctx context.Context, volumeID string, meta metad // Calculate the default next issuance time. // The implementation's writeKeypair function may override this value before // writing to the storage layer. - renewalPoint, err := calculateNextIssuanceTime(req.Status.Certificate) + expiryPoint, renewalPoint, err := getExpiryAndDefaultNextIssuanceTime(req.Status.Certificate) if err != nil { return fmt.Errorf("calculating next issuance time: %w", err) } @@ -606,6 +620,10 @@ func (m *Manager) handleRequest(ctx context.Context, volumeID string, meta metad } log.V(2).Info("Wrote new keypair to storage") + // Update the request metrics. + // Using meta.NextIssuanceTime instead of renewalPoint here, in case writeKeypair overrides the value. + m.metrics.UpdateCertificateRequest(req, expiryPoint, *meta.NextIssuanceTime) + // We must explicitly delete the private key from the pending requests map so that the existing Completed // request will not be re-used upon renewal. // Without this, the renewal would pick up the existing issued certificate and re-issue, rather than requesting @@ -657,6 +675,9 @@ func (m *Manager) cleanupStaleRequests(ctx context.Context, log logr.Logger, vol } } + // Remove the CertificateRequest from the metrics. + m.metrics.RemoveCertificateRequest(toDelete.Name, toDelete.Namespace) + log.Info("Deleted CertificateRequest resource", "name", toDelete.Name, "namespace", toDelete.Namespace) } @@ -756,6 +777,8 @@ func (m *Manager) ManageVolumeImmediate(ctx context.Context, volumeID string) (m // If issuance fails, immediately return without retrying so the caller can decide // how to proceed depending on the context this method was called within. if err := m.issue(ctx, volumeID); err != nil { + // Increase issue error count + m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) return true, err } } @@ -783,6 +806,8 @@ func (m *Manager) manageVolumeIfNotManaged(volumeID string) (managed bool) { // construct a new channel used to stop management of the volume stopCh := make(chan struct{}) m.managedVolumes[volumeID] = stopCh + // Increase managed volume count for this driver + m.metrics.IncrementManagedVolumeCount(m.nodeNameHash) return true } @@ -800,6 +825,10 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { return false } + // Increase managed certificate count for this driver. + // We assume each volume will have one certificate to be managed. + m.metrics.IncrementManagedCertificateCount(m.nodeNameHash) + // Create a context that will be cancelled when the stopCh is closed ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -835,6 +864,8 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { defer issueCancel() if err := m.issue(issueCtx, volumeID); err != nil { log.Error(err, "Failed to issue certificate, retrying after applying exponential backoff") + // Increase issue error count + m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) return false, nil } return true, nil @@ -874,6 +905,14 @@ func (m *Manager) UnmanageVolume(volumeID string) { if stopCh, ok := m.managedVolumes[volumeID]; ok { close(stopCh) delete(m.managedVolumes, volumeID) + if reqs, err := m.listAllRequestsForVolume(volumeID); err == nil { + // Remove the CertificateRequest from the metrics with the best effort. + for _, req := range reqs { + if req != nil { + m.metrics.RemoveCertificateRequest(req.Name, req.Namespace) + } + } + } } } @@ -919,19 +958,19 @@ func (m *Manager) Stop() { } } -// calculateNextIssuanceTime will return the default time at which the certificate -// should be renewed by the driver- 2/3rds through its lifetime (NotAfter - -// NotBefore). -func calculateNextIssuanceTime(chain []byte) (time.Time, error) { +// getExpiryAndDefaultNextIssuanceTime will return the certificate expiry time, together with +// default time at which the certificate should be renewed by the driver- 2/3rds through its +// lifetime (NotAfter - NotBefore). +func getExpiryAndDefaultNextIssuanceTime(chain []byte) (time.Time, time.Time, error) { block, _ := pem.Decode(chain) crt, err := x509.ParseCertificate(block.Bytes) if err != nil { - return time.Time{}, fmt.Errorf("parsing issued certificate: %w", err) + return time.Time{}, time.Time{}, fmt.Errorf("parsing issued certificate: %w", err) } actualDuration := crt.NotAfter.Sub(crt.NotBefore) renewBeforeNotAfter := actualDuration / 3 - return crt.NotAfter.Add(-renewBeforeNotAfter), nil + return crt.NotAfter, crt.NotAfter.Add(-renewBeforeNotAfter), nil } diff --git a/manager/manager_test.go b/manager/manager_test.go index c350be3..21bea15 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -470,7 +470,7 @@ func TestManager_cleanupStaleRequests(t *testing.T) { } } -func Test_calculateNextIssuanceTime(t *testing.T) { +func Test_getExpiryAndDefaultNextIssuanceTime(t *testing.T) { notBefore := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) notAfter := time.Date(1970, time.January, 4, 0, 0, 0, 0, time.UTC) pk, err := rsa.GenerateKey(rand.Reader, 2048) @@ -490,20 +490,23 @@ func Test_calculateNextIssuanceTime(t *testing.T) { certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) tests := map[string]struct { - expTime time.Time - expErr bool + expTime time.Time + renewTime time.Time + expErr bool }{ "if no attributes given, return 2/3rd certificate lifetime": { - expTime: notBefore.AddDate(0, 0, 2), - expErr: false, + expTime: notAfter, + renewTime: notBefore.AddDate(0, 0, 2), + expErr: false, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { - renewTime, err := calculateNextIssuanceTime(certPEM) + expTime, renewTime, err := getExpiryAndDefaultNextIssuanceTime(certPEM) assert.Equal(t, test.expErr, err != nil) - assert.Equal(t, test.expTime, renewTime) + assert.Equal(t, test.expTime, expTime) + assert.Equal(t, test.renewTime, renewTime) }) } } diff --git a/metrics/certificaterequest.go b/metrics/certificaterequest.go new file mode 100644 index 0000000..40a91fc --- /dev/null +++ b/metrics/certificaterequest.go @@ -0,0 +1,102 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" +) + +var readyConditionStatuses = [...]cmmeta.ConditionStatus{ + cmmeta.ConditionTrue, + cmmeta.ConditionFalse, + cmmeta.ConditionUnknown, +} + +// UpdateCertificateRequest will update the given CertificateRequest's metrics for its expiry, renewal, and status condition. +func (m *Metrics) UpdateCertificateRequest(cr *cmapi.CertificateRequest, exp, renewal time.Time) { + m.updateCertificateRequestExpiryAndRenewalTime(cr, exp, renewal) + m.updateCertificateRequestStatus(cr) +} + +// updateCertificateRequestExpiryAndRenewalTime updates the expiry and renewal time of a certificate request +func (m *Metrics) updateCertificateRequestExpiryAndRenewalTime(cr *cmapi.CertificateRequest, exp, renewal time.Time) { + expiryTime := 0.0 + if !exp.IsZero() { + expiryTime = float64(exp.Unix()) + } + m.certificateRequestExpiryTimeSeconds.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group}).Set(expiryTime) + + renewalTime := 0.0 + if !renewal.IsZero() { + renewalTime = float64(renewal.Unix()) + } + m.certificateRequestRenewalTimeSeconds.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group}).Set(renewalTime) +} + +// updateCertificateRequestStatus will update the metric for that Certificate Request +func (m *Metrics) updateCertificateRequestStatus(cr *cmapi.CertificateRequest) { + for _, c := range cr.Status.Conditions { + if c.Type == cmapi.CertificateRequestConditionReady { + m.updateCertificateRequestReadyStatus(cr, c.Status) + return + } + } + + // If no status condition set yet, set to Unknown + m.updateCertificateRequestReadyStatus(cr, cmmeta.ConditionUnknown) +} + +func (m *Metrics) updateCertificateRequestReadyStatus(cr *cmapi.CertificateRequest, current cmmeta.ConditionStatus) { + for _, condition := range readyConditionStatuses { + value := 0.0 + + if current == condition { + value = 1.0 + } + + m.certificateRequestReadyStatus.With(prometheus.Labels{ + "name": cr.Name, + "namespace": cr.Namespace, + "condition": string(condition), + "issuer_name": cr.Spec.IssuerRef.Name, + "issuer_kind": cr.Spec.IssuerRef.Kind, + "issuer_group": cr.Spec.IssuerRef.Group, + }).Set(value) + } +} + +// RemoveCertificateRequest will delete the CertificateRequest metrics from continuing to be exposed. +func (m *Metrics) RemoveCertificateRequest(name, namespace string) { + m.certificateRequestExpiryTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) + m.certificateRequestRenewalTimeSeconds.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) + m.certificateRequestReadyStatus.DeletePartialMatch(prometheus.Labels{"name": name, "namespace": namespace}) +} diff --git a/metrics/certificaterequest_test.go b/metrics/certificaterequest_test.go new file mode 100644 index 0000000..201b461 --- /dev/null +++ b/metrics/certificaterequest_test.go @@ -0,0 +1,358 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + "testing" + "time" + + "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus/testutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/cert-manager/cert-manager/test/unit/gen" +) + +const expiryMetadata = ` + # HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The date after which the certificate request expires. Expressed as a Unix Epoch Time. + # TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +` + +const renewalTimeMetadata = ` + # HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The number of seconds before expiration time the certificate request should renew. + # TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +` + +const readyMetadata = ` + # HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. + # TYPE certmanager_csi_certificate_request_ready_status gauge +` + +func TestCertificateRequestMetrics(t *testing.T) { + type testT struct { + cr *cmapi.CertificateRequest + notAfter, renewBefore time.Time + expectedExpiry, expectedReady, expectedRenewalTime string + } + tests := map[string]testT{ + "certificate with expiry and ready status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + ), + notAfter: time.Unix(2208988804, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with no expiry and no status should give an expiry of 0 and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + ), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status False should give an expiry and False status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + ), + notAfter: time.Unix(100, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 100 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and status Unknown should give an expiry and Unknown status": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + ), + notAfter: time.Unix(99999, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 99999 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + }, + "certificate with expiry and ready status and renew before": { + cr: gen.CertificateRequest("test-certificate-request", + gen.SetCertificateRequestNamespace("test-ns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + ), + notAfter: time.Unix(2208988804, 0), + renewBefore: time.Unix(2108988804, 0), + + expectedExpiry: ` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 0 +`, + expectedRenewalTime: ` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-certificate-request",namespace="test-ns"} 2.108988804e+09 +`, + }, + } + for n, test := range tests { + t.Run(n, func(t *testing.T) { + testLog := testr.New(t) + m := New(&testLog) + m.UpdateCertificateRequest(test.cr, test.notAfter, test.renewBefore) + + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+test.expectedExpiry), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestRenewalTimeSeconds, + strings.NewReader(renewalTimeMetadata+test.expectedRenewalTime), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+test.expectedReady), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + }) + } +} + +func TestCertificateRequestCache(t *testing.T) { + testLog := testr.New(t) + m := New(&testLog) + + // private key to be used to generate X509 certificate + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: "testns", Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore := time.Unix(0, 0) + notAfter1, notAfter2, notAfter3 := + time.Unix(100, 0), time.Unix(200, 0), time.Unix(300, 0) + renew1, renew2, renew3 := + time.Unix(50, 0), time.Unix(150, 0), time.Unix(250, 0) + + cr1 := gen.CertificateRequest("cr1", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionUnknown, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter1)), + ) + cr2 := gen.CertificateRequest("cr2", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionTrue, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter2)), + ) + cr3 := gen.CertificateRequest("cr3", + gen.SetCertificateRequestNamespace("testns"), + gen.SetCertificateRequestIssuer(cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }), + gen.SetCertificateRequestStatusCondition(cmapi.CertificateRequestCondition{ + Type: cmapi.CertificateRequestConditionReady, + Status: cmmeta.ConditionFalse, + }), + gen.SetCertificateRequestCertificate( + testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter3)), + ) + + // Observe all three Certificate metrics + m.UpdateCertificateRequest(cr1, notAfter1, renew1) + m.UpdateCertificateRequest(cr2, notAfter2, renew2) + m.UpdateCertificateRequest(cr3, notAfter3, renew3) + + // Check all three metrics exist + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 200 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + if err := testutil.CollectAndCompare(m.certificateRequestRenewalTimeSeconds, + strings.NewReader(renewalTimeMetadata+` + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 50 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr2",namespace="testns"} 150 + certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 250 +`), + "certmanager_csi_certificate_request_renewal_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove second certificate and check not exists + m.RemoveCertificateRequest("cr2", "testns") + if err := testutil.CollectAndCompare(m.certificateRequestReadyStatus, + strings.NewReader(readyMetadata+` + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 1 + certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 0 +`), + "certmanager_csi_certificate_request_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, + strings.NewReader(expiryMetadata+` + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr1",namespace="testns"} 100 + certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="cr3",namespace="testns"} 300 +`), + "certmanager_csi_certificate_request_expiration_timestamp_seconds", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + // Remove all Certificates (even is already removed) and observe no Certificates + m.RemoveCertificateRequest("cr1", "testns") + m.RemoveCertificateRequest("cr2", "testns") + m.RemoveCertificateRequest("cr3", "testns") + if testutil.CollectAndCount(m.certificateRequestReadyStatus, "certmanager_csi_certificate_request_ready_status") != 0 { + t.Errorf("unexpected collecting result") + } + if testutil.CollectAndCount(m.certificateRequestExpiryTimeSeconds, "certmanager_csi_certificate_request_expiration_timestamp_seconds") != 0 { + t.Errorf("unexpected collecting result") + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..e2f0b06 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "net" + "net/http" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + // Namespace is the namespace for csi-lib metric names + namespace = "certmanager" + subsystem = "csi" + prometheusMetricsServerReadTimeout = 8 * time.Second + prometheusMetricsServerWriteTimeout = 8 * time.Second + prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB +) + +// Metrics is designed to be a shared object for updating the metrics exposed by csi-lib +type Metrics struct { + log logr.Logger + registry *prometheus.Registry + + certificateRequestExpiryTimeSeconds *prometheus.GaugeVec + certificateRequestRenewalTimeSeconds *prometheus.GaugeVec + certificateRequestReadyStatus *prometheus.GaugeVec + driverIssueCallCount *prometheus.CounterVec + driverIssueErrorCount *prometheus.CounterVec + managedVolumeCount *prometheus.CounterVec + managedCertificateCount *prometheus.CounterVec +} + +// New creates a Metrics struct and populates it with prometheus metric types. +func New(logger *logr.Logger) *Metrics { + var ( + certificateRequestExpiryTimeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_expiration_timestamp_seconds", + Help: "The date after which the certificate request expires. Expressed as a Unix Epoch Time.", + }, + []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + certificateRequestRenewalTimeSeconds = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_renewal_timestamp_seconds", + Help: "The number of seconds before expiration time the certificate request should renew.", + }, + []string{"name", "namespace", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + certificateRequestReadyStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "certificate_request_ready_status", + Help: "The ready status of the certificate request.", + }, + []string{"name", "namespace", "condition", "issuer_name", "issuer_kind", "issuer_group"}, + ) + + driverIssueCallCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_call_count", + Help: "The number of issue() calls made by the driver.", + }, + []string{"node", "volume"}, + ) + + driverIssueErrorCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "driver_issue_error_count", + Help: "The number of errors encountered during the driver issue() calls.", + }, + []string{"node", "volume"}, + ) + + managedVolumeCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "managed_volume_count", + Help: "The number of volume managed by the csi driver.", + }, + []string{"node"}, + ) + + managedCertificateCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "managed_certificate_count", + Help: "The number of certificates managed by the csi driver.", + }, + []string{"node"}, + ) + ) + + // Create Registry and register the recommended collectors + registry := prometheus.NewRegistry() + registry.MustRegister( + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + ) + // Create server and register Prometheus metrics handler + m := &Metrics{ + log: logger.WithName("metrics"), + registry: registry, + + certificateRequestExpiryTimeSeconds: certificateRequestExpiryTimeSeconds, + certificateRequestRenewalTimeSeconds: certificateRequestRenewalTimeSeconds, + certificateRequestReadyStatus: certificateRequestReadyStatus, + driverIssueCallCount: driverIssueCallCount, + driverIssueErrorCount: driverIssueErrorCount, + managedVolumeCount: managedVolumeCount, + managedCertificateCount: managedCertificateCount, + } + + return m +} + +// NewServer registers Prometheus metrics and returns a new Prometheus metrics HTTP server. +func (m *Metrics) NewServer(ln net.Listener) *http.Server { + m.registry.MustRegister(m.certificateRequestExpiryTimeSeconds) + m.registry.MustRegister(m.certificateRequestRenewalTimeSeconds) + m.registry.MustRegister(m.certificateRequestReadyStatus) + m.registry.MustRegister(m.driverIssueCallCount) + m.registry.MustRegister(m.driverIssueErrorCount) + m.registry.MustRegister(m.managedVolumeCount) + m.registry.MustRegister(m.managedCertificateCount) + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + + server := &http.Server{ + Addr: ln.Addr().String(), + ReadTimeout: prometheusMetricsServerReadTimeout, + WriteTimeout: prometheusMetricsServerWriteTimeout, + MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes, + Handler: mux, + } + + return server +} + +// IncrementIssueCallCount will increase the issue call counter for the driver. +func (m *Metrics) IncrementIssueCallCount(nodeNameHash, volumeID string) { + m.driverIssueCallCount.WithLabelValues(nodeNameHash, volumeID).Inc() +} + +// IncrementIssueErrorCount will increase count of errors during issue call of the driver. +func (m *Metrics) IncrementIssueErrorCount(nodeNameHash, volumeID string) { + m.driverIssueErrorCount.WithLabelValues(nodeNameHash, volumeID).Inc() +} + +// IncrementManagedVolumeCount will increase the managed volume counter for the driver. +func (m *Metrics) IncrementManagedVolumeCount(nodeNameHash string) { + m.managedVolumeCount.WithLabelValues(nodeNameHash).Inc() +} + +// IncrementManagedCertificateCount will increase the managed certificate count for the driver. +func (m *Metrics) IncrementManagedCertificateCount(nodeNameHash string) { + m.managedCertificateCount.WithLabelValues(nodeNameHash).Inc() +} diff --git a/test/driver/driver_testing.go b/test/driver/driver_testing.go index bbd4252..02659b2 100644 --- a/test/driver/driver_testing.go +++ b/test/driver/driver_testing.go @@ -36,6 +36,7 @@ import ( "github.com/cert-manager/csi-lib/driver" "github.com/cert-manager/csi-lib/manager" "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" "github.com/cert-manager/csi-lib/storage" ) @@ -45,6 +46,7 @@ type Options struct { Log *logr.Logger Client cmclient.Interface Mounter mount.Interface + Metrics *metrics.Metrics NodeID string MaxRequestsPerVolume int @@ -109,6 +111,7 @@ func Run(t *testing.T, opts Options) (Options, csi.NodeClient, func()) { Clock: opts.Clock, Log: opts.Log, NodeID: opts.NodeID, + Metrics: opts.Metrics, MaxRequestsPerVolume: opts.MaxRequestsPerVolume, GeneratePrivateKey: opts.GeneratePrivateKey, GenerateRequest: opts.GenerateRequest, diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go new file mode 100644 index 0000000..34a3923 --- /dev/null +++ b/test/integration/metrics_test.go @@ -0,0 +1,235 @@ +package integration + +import ( + "context" + "crypto" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/go-logr/logr/testr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + fakeclock "k8s.io/utils/clock/testing" + + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + testcrypto "github.com/cert-manager/cert-manager/test/unit/crypto" + "github.com/cert-manager/csi-lib/manager" + "github.com/cert-manager/csi-lib/metadata" + "github.com/cert-manager/csi-lib/metrics" + "github.com/cert-manager/csi-lib/storage" + testdriver "github.com/cert-manager/csi-lib/test/driver" + testutil "github.com/cert-manager/csi-lib/test/util" +) + +var ( + testMetrics = func(ctx context.Context, metricsEndpoint, expectedOutput string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metricsEndpoint, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + output, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + trimmedOutput := strings.SplitN(string(output), "# HELP go_gc_duration_seconds", 2)[0] + if strings.TrimSpace(trimmedOutput) != strings.TrimSpace(expectedOutput) { + return fmt.Errorf("got unexpected metrics output\nexp:\n%s\ngot:\n%s\n", + expectedOutput, trimmedOutput) + } + + return nil + } + + waitForMetrics = func(t *testing.T, ctx context.Context, metricsEndpoint, expectedOutput string) { + var lastErr error + err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { + if err := testMetrics(ctx, metricsEndpoint, expectedOutput); err != nil { + lastErr = err + return false, nil + } + + return true, nil + }) + if err != nil { + t.Fatalf("%s: failed to wait for expected metrics to be exposed: %s", err, lastErr) + } + } +) + +func TestMetricsServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + testLog := testr.New(t) + testNamespace := "test-ns" + + // Build metrics handler, and start metrics server with a random available port + metricsHandler := metrics.New(&testLog) + metricsLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + metricsServer := metricsHandler.NewServer(metricsLn) + errCh := make(chan error) + go func() { + defer close(errCh) + testLog.Info("starting metrics server", "address", metricsLn.Addr()) + if err := metricsServer.Serve(metricsLn); err != http.ErrServerClosed { + errCh <- err + } + }() + defer func() { + // allow a timeout for graceful shutdown + shutdownCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + if err := metricsServer.Shutdown(shutdownCtx); err != nil { + t.Fatal(err) + } + err := <-errCh + if err != nil { + t.Fatal(err) + } + }() + + // Build and start the driver + store := storage.NewMemoryFS() + clock := fakeclock.NewFakeClock(time.Now()) + opts, cl, stop := testdriver.Run(t, testdriver.Options{ + Store: store, + Clock: clock, + Metrics: metricsHandler, + Log: &testLog, + GeneratePrivateKey: func(meta metadata.Metadata) (crypto.PrivateKey, error) { + return nil, nil + }, + GenerateRequest: func(meta metadata.Metadata) (*manager.CertificateRequestBundle, error) { + return &manager.CertificateRequestBundle{ + Namespace: testNamespace, + IssuerRef: cmmeta.ObjectReference{ + Name: "test-issuer", + Kind: "test-issuer-kind", + Group: "test-issuer-group", + }, + }, nil + }, + SignRequest: func(meta metadata.Metadata, key crypto.PrivateKey, request *x509.CertificateRequest) (csr []byte, err error) { + return []byte{}, nil + }, + WriteKeypair: func(meta metadata.Metadata, key crypto.PrivateKey, chain []byte, ca []byte) error { + store.WriteFiles(meta, map[string][]byte{ + "ca": ca, + "cert": chain, + }) + nextIssuanceTime := clock.Now().Add(time.Hour) + meta.NextIssuanceTime = &nextIssuanceTime + return store.WriteMetadata(meta.VolumeID, meta) + }, + }) + defer stop() + + // Should expose no additional metrics + metricsEndpoint := fmt.Sprintf("http://%s/metrics", metricsServer.Addr) + waitForMetrics(t, ctx, metricsEndpoint, "") + + // Create a self-signed Certificate and wait for it to be issued + privKey := testcrypto.MustCreatePEMPrivateKey(t) + certTemplate := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "test"}, + Spec: cmapi.CertificateSpec{ + CommonName: "test.example.com", + }, + } + notBefore, notAfter := time.Unix(0, 0), time.Unix(300, 0) // renewal time will be 200 + selfSignedCertBytesWithValidity := testcrypto.MustCreateCertWithNotBeforeAfter(t, privKey, certTemplate, notBefore, notAfter) + go testutil.IssueOneRequest(ctx, t, opts.Client, testNamespace, selfSignedCertBytesWithValidity, []byte("ca bytes")) + + // Spin up a test pod + tmpDir, err := os.MkdirTemp("", "*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + _, err = cl.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + VolumeContext: map[string]string{ + "csi.storage.k8s.io/ephemeral": "true", + "csi.storage.k8s.io/pod.name": "the-pod-name", + "csi.storage.k8s.io/pod.namespace": testNamespace, + }, + TargetPath: tmpDir, + Readonly: true, + }) + if err != nil { + t.Fatal(err) + } + + // Get the CSR name + req, err := testutil.WaitAndGetOneCertificateRequestInNamespace(ctx, opts.Client, testNamespace) + if err != nil { + t.Fatal(err) + } + + // Should expose that CertificateRequest as ready with expiry and renewal time + // node="f56fd9f8b" is the hash value of "test-node" defined in driver_testing.go + expectedOutputTemplate := `# HELP certmanager_csi_certificate_request_expiration_timestamp_seconds The date after which the certificate request expires. Expressed as a Unix Epoch Time. +# TYPE certmanager_csi_certificate_request_expiration_timestamp_seconds gauge +certmanager_csi_certificate_request_expiration_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 300 +# HELP certmanager_csi_certificate_request_ready_status The ready status of the certificate request. +# TYPE certmanager_csi_certificate_request_ready_status gauge +certmanager_csi_certificate_request_ready_status{condition="False",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +certmanager_csi_certificate_request_ready_status{condition="True",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 1 +certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 0 +# HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The number of seconds before expiration time the certificate request should renew. +# TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge +certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 200 +# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count counter +certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count counter +certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count counter +certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 +` + waitForMetrics(t, ctx, metricsEndpoint, strings.ReplaceAll(expectedOutputTemplate, "test-cr-name", req.Name)) + + // Delete the test pod + _, err = cl.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{ + VolumeId: "test-vol", + TargetPath: tmpDir, + }) + if err != nil { + t.Fatal(err) + } + + // Should expose no CertificateRequest and only metrics counters + waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count counter +certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count counter +certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count counter +certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 +`) + +} diff --git a/test/util/testutil.go b/test/util/testutil.go index d405e4a..6d9dd95 100644 --- a/test/util/testutil.go +++ b/test/util/testutil.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { +func WaitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmclient.Interface, ns string) (*cmapi.CertificateRequest, error) { var req *cmapi.CertificateRequest if err := wait.PollUntilContextCancel(ctx, time.Millisecond*50, true, func(ctx context.Context) (done bool, err error) { reqs, err := client.CertmanagerV1().CertificateRequests(ns).List(ctx, metav1.ListOptions{}) @@ -53,7 +53,7 @@ func waitAndGetOneCertificateRequestInNamespace(ctx context.Context, client cmcl func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, cert, ca []byte) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err } @@ -80,7 +80,7 @@ func IssueOneRequest(ctx context.Context, t *testing.T, client cmclient.Interfac func SetCertificateRequestConditions(ctx context.Context, t *testing.T, client cmclient.Interface, namespace string, conditions ...cmapi.CertificateRequestCondition) { if err := func() error { - req, err := waitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) + req, err := WaitAndGetOneCertificateRequestInNamespace(ctx, client, namespace) if err != nil { return err } From 104e7b03e532c89ab4341b28d1917fbfea9ea78a Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Tue, 4 Feb 2025 16:23:02 -0800 Subject: [PATCH 2/3] Resolve comments for metrics package fixes: #60 Signed-off-by: Jing Liu --- .DS_Store | Bin 0 -> 8196 bytes go.mod | 4 ++ go.sum | 72 +++++++++++++++++++++++++++++ manager/manager.go | 3 +- metrics/certificaterequest_test.go | 5 +- metrics/metrics.go | 38 ++++----------- test/integration/metrics_test.go | 12 ++++- 7 files changed, 100 insertions(+), 34 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..fcabe824e9c646fabb048f1fb5a47f2ba38c8687 GIT binary patch literal 8196 zcmeHMziSjh6n+!W+(8f`DH5?+5Cb-9#6qw+!`TUf30SDnO)ezSJ2$@+jSx-|6toC} zjg_U9*l8Is_WlE+;9n37*jV_znPKm}{c)BNZU%PWvhSPuzWHWuCbJoaZZ72K<||hz9kkwJY7HLYRvt;|XO+q+YnzXmLST30$ylrP|A?W-7p`xt>$%{~1S+$t~mKVI5tuDJ2f zV7mP;!)P5H&`=fp4=AQKhNVUGTFLlE9wD_Hy1aMe+7atTrXA@>IZEp}&3Y=d2sLwb zM=Lq=L*}H0ubb1|@$hvedr@BVZZU5SV{w<(c{C7XYH9w#b`eJE*m#=i$h9T)^ulRf z5$mdAWMZpmAvaUZ5Ua`{q^9%dwthczb0vrB&X#hN(&KjX8uZM=rx^1$=BRzD9)+SD zMryk9Uig%0OZwCcr*yeLTB@`LH3>#9#(J8W$1{=jNg<@Bz0awpOgmChIZErX>s!Un zB`|7#p15_L=NWQR!=K%BT}$gq_M*Jz{eVu=5O^0cmsc@9H}RW}@wA>@Vc(6HBc-P8 zZ>cXO?Mj9JMi=O1&HY#oxK5Ap`Tt`1_y2yhCiFQt&^Hd4N@J=qiSGKdb*|%l){b!9 z!o`Jg9T#mC-0X22(H_SU(|;J^IRcdH)P3x@h&L$v@FAeqYhRbUNiUqr_BD9ziJ;Q>i_@% literal 0 HcmV?d00001 diff --git a/go.mod b/go.mod index f96bf21..c8c4501 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( ) require ( + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -25,6 +26,8 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect + github.com/go-ldap/ldap/v3 v3.4.8 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -59,6 +62,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect diff --git a/go.sum b/go.sum index c99fed8..bc61380 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -19,6 +23,11 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.6 h1:CYsqysemXfEaQbyrLJmdsCRuufHoLa3P/gGWGl5TDrM= +github.com/go-asn1-ber/asn1-ber v1.5.6/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ= +github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -50,8 +59,25 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -110,14 +136,21 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -127,14 +160,30 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= @@ -142,16 +191,36 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= @@ -160,6 +229,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -179,6 +250,7 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/manager/manager.go b/manager/manager.go index c8c583e..a1e29d3 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -34,6 +34,7 @@ import ( cminformers "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions" cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -131,7 +132,7 @@ func NewManager(opts Options) (*Manager, error) { return nil, errors.New("log must be set") } if opts.Metrics == nil { - opts.Metrics = metrics.New(opts.Log) + opts.Metrics = metrics.New(opts.Log, prometheus.NewRegistry()) } if opts.MetadataReader == nil { return nil, errors.New("MetadataReader must be set") diff --git a/metrics/certificaterequest_test.go b/metrics/certificaterequest_test.go index 201b461..1ec7f5b 100644 --- a/metrics/certificaterequest_test.go +++ b/metrics/certificaterequest_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -188,7 +189,7 @@ func TestCertificateRequestMetrics(t *testing.T) { for n, test := range tests { t.Run(n, func(t *testing.T) { testLog := testr.New(t) - m := New(&testLog) + m := New(&testLog, prometheus.NewRegistry()) m.UpdateCertificateRequest(test.cr, test.notAfter, test.renewBefore) if err := testutil.CollectAndCompare(m.certificateRequestExpiryTimeSeconds, @@ -217,7 +218,7 @@ func TestCertificateRequestMetrics(t *testing.T) { func TestCertificateRequestCache(t *testing.T) { testLog := testr.New(t) - m := New(&testLog) + m := New(&testLog, prometheus.NewRegistry()) // private key to be used to generate X509 certificate privKey := testcrypto.MustCreatePEMPrivateKey(t) diff --git a/metrics/metrics.go b/metrics/metrics.go index e2f0b06..f686ebe 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -17,23 +17,17 @@ limitations under the License. package metrics import ( - "net" "net/http" - "time" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( // Namespace is the namespace for csi-lib metric names - namespace = "certmanager" - subsystem = "csi" - prometheusMetricsServerReadTimeout = 8 * time.Second - prometheusMetricsServerWriteTimeout = 8 * time.Second - prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB + namespace = "certmanager" + subsystem = "csi" ) // Metrics is designed to be a shared object for updating the metrics exposed by csi-lib @@ -51,7 +45,7 @@ type Metrics struct { } // New creates a Metrics struct and populates it with prometheus metric types. -func New(logger *logr.Logger) *Metrics { +func New(logger *logr.Logger, registry *prometheus.Registry) *Metrics { var ( certificateRequestExpiryTimeSeconds = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -124,12 +118,6 @@ func New(logger *logr.Logger) *Metrics { ) ) - // Create Registry and register the recommended collectors - registry := prometheus.NewRegistry() - registry.MustRegister( - collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - collectors.NewGoCollector(), - ) // Create server and register Prometheus metrics handler m := &Metrics{ log: logger.WithName("metrics"), @@ -144,11 +132,6 @@ func New(logger *logr.Logger) *Metrics { managedCertificateCount: managedCertificateCount, } - return m -} - -// NewServer registers Prometheus metrics and returns a new Prometheus metrics HTTP server. -func (m *Metrics) NewServer(ln net.Listener) *http.Server { m.registry.MustRegister(m.certificateRequestExpiryTimeSeconds) m.registry.MustRegister(m.certificateRequestRenewalTimeSeconds) m.registry.MustRegister(m.certificateRequestReadyStatus) @@ -157,18 +140,15 @@ func (m *Metrics) NewServer(ln net.Listener) *http.Server { m.registry.MustRegister(m.managedVolumeCount) m.registry.MustRegister(m.managedCertificateCount) + return m +} + +// DefaultHandler returns a default prometheus metrics HTTP handler +func (m *Metrics) DefaultHandler() http.Handler { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) - server := &http.Server{ - Addr: ln.Addr().String(), - ReadTimeout: prometheusMetricsServerReadTimeout, - WriteTimeout: prometheusMetricsServerWriteTimeout, - MaxHeaderBytes: prometheusMetricsServerMaxHeaderBytes, - Handler: mux, - } - - return server + return mux } // IncrementIssueCallCount will increase the issue call counter for the driver. diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go index 34a3923..f578fc2 100644 --- a/test/integration/metrics_test.go +++ b/test/integration/metrics_test.go @@ -15,6 +15,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/go-logr/logr/testr" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" fakeclock "k8s.io/utils/clock/testing" @@ -80,12 +81,19 @@ func TestMetricsServer(t *testing.T) { testNamespace := "test-ns" // Build metrics handler, and start metrics server with a random available port - metricsHandler := metrics.New(&testLog) + metricsHandler := metrics.New(&testLog, prometheus.NewRegistry()) metricsLn, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } - metricsServer := metricsHandler.NewServer(metricsLn) + metricsServer := &http.Server{ + Addr: metricsLn.Addr().String(), + ReadTimeout: 8 * time.Second, + WriteTimeout: 8 * time.Second, + MaxHeaderBytes: 1 << 20, // 1 MiB + Handler: metricsHandler.DefaultHandler(), + } + errCh := make(chan error) go func() { defer close(errCh) From b9186bad5b6f9af9bc93dbd28571d2d9219700e6 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Fri, 30 May 2025 12:01:48 -0700 Subject: [PATCH 3/3] fix promlinter and boilerplate issues Signed-off-by: Jing Liu --- manager/manager.go | 10 ++--- metrics/metrics.go | 64 ++++++++++++++++---------------- test/integration/metrics_test.go | 52 +++++++++++++++++--------- 3 files changed, 71 insertions(+), 55 deletions(-) diff --git a/manager/manager.go b/manager/manager.go index a1e29d3..77beb9a 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -400,7 +400,7 @@ func (m *Manager) issue(ctx context.Context, volumeID string) error { log.Info("Processing issuance") // Increase issue count - m.metrics.IncrementIssueCallCount(m.nodeNameHash, volumeID) + m.metrics.IncrementIssueCallCountTotal(m.nodeNameHash, volumeID) if err := m.cleanupStaleRequests(ctx, log, volumeID); err != nil { return fmt.Errorf("cleaning up stale requests: %w", err) @@ -779,7 +779,7 @@ func (m *Manager) ManageVolumeImmediate(ctx context.Context, volumeID string) (m // how to proceed depending on the context this method was called within. if err := m.issue(ctx, volumeID); err != nil { // Increase issue error count - m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) + m.metrics.IncrementIssueErrorCountTotal(m.nodeNameHash, volumeID) return true, err } } @@ -808,7 +808,7 @@ func (m *Manager) manageVolumeIfNotManaged(volumeID string) (managed bool) { stopCh := make(chan struct{}) m.managedVolumes[volumeID] = stopCh // Increase managed volume count for this driver - m.metrics.IncrementManagedVolumeCount(m.nodeNameHash) + m.metrics.IncrementManagedVolumeCountTotal(m.nodeNameHash) return true } @@ -828,7 +828,7 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { // Increase managed certificate count for this driver. // We assume each volume will have one certificate to be managed. - m.metrics.IncrementManagedCertificateCount(m.nodeNameHash) + m.metrics.IncrementManagedCertificateCountTotal(m.nodeNameHash) // Create a context that will be cancelled when the stopCh is closed ctx, cancel := context.WithCancel(context.Background()) @@ -866,7 +866,7 @@ func (m *Manager) startRenewalRoutine(volumeID string) (started bool) { if err := m.issue(issueCtx, volumeID); err != nil { log.Error(err, "Failed to issue certificate, retrying after applying exponential backoff") // Increase issue error count - m.metrics.IncrementIssueErrorCount(m.nodeNameHash, volumeID) + m.metrics.IncrementIssueErrorCountTotal(m.nodeNameHash, volumeID) return false, nil } return true, nil diff --git a/metrics/metrics.go b/metrics/metrics.go index f686ebe..47bbc2f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -38,10 +38,10 @@ type Metrics struct { certificateRequestExpiryTimeSeconds *prometheus.GaugeVec certificateRequestRenewalTimeSeconds *prometheus.GaugeVec certificateRequestReadyStatus *prometheus.GaugeVec - driverIssueCallCount *prometheus.CounterVec - driverIssueErrorCount *prometheus.CounterVec - managedVolumeCount *prometheus.CounterVec - managedCertificateCount *prometheus.CounterVec + driverIssueCallCountTotal *prometheus.CounterVec + driverIssueErrorCountTotal *prometheus.CounterVec + managedVolumeCountTotal *prometheus.CounterVec + managedCertificateCountTotal *prometheus.CounterVec } // New creates a Metrics struct and populates it with prometheus metric types. @@ -77,41 +77,41 @@ func New(logger *logr.Logger, registry *prometheus.Registry) *Metrics { []string{"name", "namespace", "condition", "issuer_name", "issuer_kind", "issuer_group"}, ) - driverIssueCallCount = prometheus.NewCounterVec( + driverIssueCallCountTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "driver_issue_call_count", + Name: "driver_issue_call_count_total", Help: "The number of issue() calls made by the driver.", }, []string{"node", "volume"}, ) - driverIssueErrorCount = prometheus.NewCounterVec( + driverIssueErrorCountTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "driver_issue_error_count", + Name: "driver_issue_error_count_total", Help: "The number of errors encountered during the driver issue() calls.", }, []string{"node", "volume"}, ) - managedVolumeCount = prometheus.NewCounterVec( + managedVolumeCountTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "managed_volume_count", + Name: "managed_volume_count_total", Help: "The number of volume managed by the csi driver.", }, []string{"node"}, ) - managedCertificateCount = prometheus.NewCounterVec( + managedCertificateCountTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "managed_certificate_count", + Name: "managed_certificate_count_total", Help: "The number of certificates managed by the csi driver.", }, []string{"node"}, @@ -126,19 +126,19 @@ func New(logger *logr.Logger, registry *prometheus.Registry) *Metrics { certificateRequestExpiryTimeSeconds: certificateRequestExpiryTimeSeconds, certificateRequestRenewalTimeSeconds: certificateRequestRenewalTimeSeconds, certificateRequestReadyStatus: certificateRequestReadyStatus, - driverIssueCallCount: driverIssueCallCount, - driverIssueErrorCount: driverIssueErrorCount, - managedVolumeCount: managedVolumeCount, - managedCertificateCount: managedCertificateCount, + driverIssueCallCountTotal: driverIssueCallCountTotal, + driverIssueErrorCountTotal: driverIssueErrorCountTotal, + managedVolumeCountTotal: managedVolumeCountTotal, + managedCertificateCountTotal: managedCertificateCountTotal, } m.registry.MustRegister(m.certificateRequestExpiryTimeSeconds) m.registry.MustRegister(m.certificateRequestRenewalTimeSeconds) m.registry.MustRegister(m.certificateRequestReadyStatus) - m.registry.MustRegister(m.driverIssueCallCount) - m.registry.MustRegister(m.driverIssueErrorCount) - m.registry.MustRegister(m.managedVolumeCount) - m.registry.MustRegister(m.managedCertificateCount) + m.registry.MustRegister(m.driverIssueCallCountTotal) + m.registry.MustRegister(m.driverIssueErrorCountTotal) + m.registry.MustRegister(m.managedVolumeCountTotal) + m.registry.MustRegister(m.managedCertificateCountTotal) return m } @@ -151,22 +151,22 @@ func (m *Metrics) DefaultHandler() http.Handler { return mux } -// IncrementIssueCallCount will increase the issue call counter for the driver. -func (m *Metrics) IncrementIssueCallCount(nodeNameHash, volumeID string) { - m.driverIssueCallCount.WithLabelValues(nodeNameHash, volumeID).Inc() +// IncrementIssueCallCountTotal will increase the issue call counter for the driver. +func (m *Metrics) IncrementIssueCallCountTotal(nodeNameHash, volumeID string) { + m.driverIssueCallCountTotal.WithLabelValues(nodeNameHash, volumeID).Inc() } -// IncrementIssueErrorCount will increase count of errors during issue call of the driver. -func (m *Metrics) IncrementIssueErrorCount(nodeNameHash, volumeID string) { - m.driverIssueErrorCount.WithLabelValues(nodeNameHash, volumeID).Inc() +// IncrementIssueErrorCountTotal will increase count of errors during issue call of the driver. +func (m *Metrics) IncrementIssueErrorCountTotal(nodeNameHash, volumeID string) { + m.driverIssueErrorCountTotal.WithLabelValues(nodeNameHash, volumeID).Inc() } -// IncrementManagedVolumeCount will increase the managed volume counter for the driver. -func (m *Metrics) IncrementManagedVolumeCount(nodeNameHash string) { - m.managedVolumeCount.WithLabelValues(nodeNameHash).Inc() +// IncrementManagedVolumeCountTotal will increase the managed volume counter for the driver. +func (m *Metrics) IncrementManagedVolumeCountTotal(nodeNameHash string) { + m.managedVolumeCountTotal.WithLabelValues(nodeNameHash).Inc() } -// IncrementManagedCertificateCount will increase the managed certificate count for the driver. -func (m *Metrics) IncrementManagedCertificateCount(nodeNameHash string) { - m.managedCertificateCount.WithLabelValues(nodeNameHash).Inc() +// IncrementManagedCertificateCountTotal will increase the managed certificate count for the driver. +func (m *Metrics) IncrementManagedCertificateCountTotal(nodeNameHash string) { + m.managedCertificateCountTotal.WithLabelValues(nodeNameHash).Inc() } diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go index f578fc2..0facffe 100644 --- a/test/integration/metrics_test.go +++ b/test/integration/metrics_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2025 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package integration import ( @@ -207,15 +223,15 @@ certmanager_csi_certificate_request_ready_status{condition="Unknown",issuer_grou # HELP certmanager_csi_certificate_request_renewal_timestamp_seconds The number of seconds before expiration time the certificate request should renew. # TYPE certmanager_csi_certificate_request_renewal_timestamp_seconds gauge certmanager_csi_certificate_request_renewal_timestamp_seconds{issuer_group="test-issuer-group",issuer_kind="test-issuer-kind",issuer_name="test-issuer",name="test-cr-name",namespace="test-ns"} 200 -# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. -# TYPE certmanager_csi_driver_issue_call_count counter -certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 -# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. -# TYPE certmanager_csi_managed_certificate_count counter -certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 -# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. -# TYPE certmanager_csi_managed_volume_count counter -certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 +# HELP certmanager_csi_driver_issue_call_count_total The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count_total counter +certmanager_csi_driver_issue_call_count_total{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count_total The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count_total counter +certmanager_csi_managed_certificate_count_total{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count_total The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count_total counter +certmanager_csi_managed_volume_count_total{node="f56fd9f8b"} 1 ` waitForMetrics(t, ctx, metricsEndpoint, strings.ReplaceAll(expectedOutputTemplate, "test-cr-name", req.Name)) @@ -229,15 +245,15 @@ certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 } // Should expose no CertificateRequest and only metrics counters - waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_driver_issue_call_count The number of issue() calls made by the driver. -# TYPE certmanager_csi_driver_issue_call_count counter -certmanager_csi_driver_issue_call_count{node="f56fd9f8b",volume="test-vol"} 1 -# HELP certmanager_csi_managed_certificate_count The number of certificates managed by the csi driver. -# TYPE certmanager_csi_managed_certificate_count counter -certmanager_csi_managed_certificate_count{node="f56fd9f8b"} 1 -# HELP certmanager_csi_managed_volume_count The number of volume managed by the csi driver. -# TYPE certmanager_csi_managed_volume_count counter -certmanager_csi_managed_volume_count{node="f56fd9f8b"} 1 + waitForMetrics(t, ctx, metricsEndpoint, `# HELP certmanager_csi_driver_issue_call_count_total The number of issue() calls made by the driver. +# TYPE certmanager_csi_driver_issue_call_count_total counter +certmanager_csi_driver_issue_call_count_total{node="f56fd9f8b",volume="test-vol"} 1 +# HELP certmanager_csi_managed_certificate_count_total The number of certificates managed by the csi driver. +# TYPE certmanager_csi_managed_certificate_count_total counter +certmanager_csi_managed_certificate_count_total{node="f56fd9f8b"} 1 +# HELP certmanager_csi_managed_volume_count_total The number of volume managed by the csi driver. +# TYPE certmanager_csi_managed_volume_count_total counter +certmanager_csi_managed_volume_count_total{node="f56fd9f8b"} 1 `) }