diff --git a/components/metrics/labels.go b/components/metrics/labels.go index 6b928c9c0..b4c712630 100644 --- a/components/metrics/labels.go +++ b/components/metrics/labels.go @@ -4,8 +4,6 @@ import ( "context" "github.com/ThreeDotsLabs/watermill/message" - - "github.com/prometheus/client_golang/prometheus" ) const ( @@ -26,7 +24,7 @@ var ( } ) -func labelsFromCtx(ctx context.Context, labels ...string) prometheus.Labels { +func labelsFromCtx(ctx context.Context, labels ...string) map[string]string { ctxLabels := map[string]string{} for _, l := range labels { diff --git a/components/metrics/opentelemetry_builder.go b/components/metrics/opentelemetry_builder.go new file mode 100644 index 000000000..428af2a86 --- /dev/null +++ b/components/metrics/opentelemetry_builder.go @@ -0,0 +1,89 @@ +package metrics + +import ( + "github.com/ThreeDotsLabs/watermill/internal" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/metric" +) + +func NewOpenTelemetryMetricsBuilder(meter metric.Meter, namespace string, subsystem string) OpenTelemetryMetricsBuilder { + return OpenTelemetryMetricsBuilder{ + Namespace: namespace, + Subsystem: subsystem, + meter: meter, + } +} + +// OpenTelemetryMetricsBuilder provides methods to decorate publishers, subscribers and handlers. +type OpenTelemetryMetricsBuilder struct { + meter metric.Meter + + Namespace string + Subsystem string + // PublishBuckets defines the histogram buckets for publish time histogram, defaulted if nil. + PublishBuckets []float64 + // HandlerBuckets defines the histogram buckets for handle execution time histogram, defaulted to watermill's default. + HandlerBuckets []float64 +} + +// AddOpenTelemetryRouterMetrics is a convenience function that acts on the message router to add the metrics middleware +// to all its handlers. The handlers' publishers and subscribers are also decorated. +func (b OpenTelemetryMetricsBuilder) AddOpenTelemetryRouterMetrics(r *message.Router) { + r.AddPublisherDecorators(b.DecoratePublisher) + r.AddSubscriberDecorators(b.DecorateSubscriber) + r.AddMiddleware(b.NewRouterMiddleware().Middleware) +} + +// DecoratePublisher wraps the underlying publisher with OpenTelemetry metrics. +func (b OpenTelemetryMetricsBuilder) DecoratePublisher(pub message.Publisher) (message.Publisher, error) { + var err error + d := PublisherOpenTelemetryMetricsDecorator{ + pub: pub, + publisherName: internal.StructName(pub), + } + + d.publishTimeSeconds, err = b.meter.Float64Histogram( + b.name("publish_time_seconds"), + metric.WithUnit("seconds"), + metric.WithDescription("The time that a publishing attempt (success or not) took in seconds"), + metric.WithExplicitBucketBoundaries(b.PublishBuckets...), + ) + + if err != nil { + return nil, errors.Wrap(err, "could not register publish time metric") + } + return d, nil +} + +// DecorateSubscriber wraps the underlying subscriber with OpenTelemetry metrics. +func (b OpenTelemetryMetricsBuilder) DecorateSubscriber(sub message.Subscriber) (message.Subscriber, error) { + var err error + d := &SubscriberOpenTelemetryMetricsDecorator{ + subscriberName: internal.StructName(sub), + } + + d.subscriberMessagesReceivedTotal, err = b.meter.Int64Counter( + b.name("subscriber_messages_received_total"), + metric.WithDescription("The total number of messages received by the subscriber"), + ) + if err != nil { + return nil, errors.Wrap(err, "could not register time to ack metric") + } + + d.Subscriber, err = message.MessageTransformSubscriberDecorator(d.recordMetrics)(sub) + if err != nil { + return nil, errors.Wrap(err, "could not decorate subscriber with metrics decorator") + } + + return d, nil +} +func (b OpenTelemetryMetricsBuilder) name(name string) string { + if b.Subsystem != "" { + name = b.Subsystem + "_" + name + } + if b.Namespace != "" { + name = b.Namespace + "_" + name + } + return name +} diff --git a/components/metrics/opentelemetry_handler.go b/components/metrics/opentelemetry_handler.go new file mode 100644 index 000000000..c8155f3a9 --- /dev/null +++ b/components/metrics/opentelemetry_handler.go @@ -0,0 +1,86 @@ +package metrics + +import ( + "time" + + "github.com/ThreeDotsLabs/watermill/message" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + handlerLabelKeys = []string{ + labelKeyHandlerName, + labelSuccess, + } + + // defaultHandlerExecutionTimeBuckets are one order of magnitude smaller than default buckets (5ms~10s), + // because the handler execution times are typically shorter (µs~ms range). + defaultHandlerExecutionTimeBuckets = []float64{ + 0.0005, + 0.001, + 0.0025, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + } +) + +// HandlerOpenTelemetryMetricsMiddleware is a middleware that captures OpenTelemetry metrics. +type HandlerOpenTelemetryMetricsMiddleware struct { + handlerExecutionTimeSeconds metric.Float64Histogram +} + +// Middleware returns the middleware ready to be used with watermill's Router. +func (m HandlerOpenTelemetryMetricsMiddleware) Middleware(h message.HandlerFunc) message.HandlerFunc { + return func(msg *message.Message) (msgs []*message.Message, err error) { + now := time.Now() + ctx := msg.Context() + labels := []attribute.KeyValue{ + attribute.String(labelKeyHandlerName, message.HandlerNameFromCtx(ctx)), + } + + defer func() { + if err != nil { + labels = append(labels, attribute.String(labelSuccess, "false")) + } else { + labels = append(labels, attribute.String(labelSuccess, "true")) + } + m.handlerExecutionTimeSeconds.Record( + ctx, + time.Since(now).Seconds(), + metric.WithAttributes(labels...), + ) + }() + + return h(msg) + } +} + +// NewRouterMiddleware returns new middleware. +func (b OpenTelemetryMetricsBuilder) NewRouterMiddleware() HandlerOpenTelemetryMetricsMiddleware { + var err error + m := HandlerOpenTelemetryMetricsMiddleware{} + + if b.HandlerBuckets == nil { + b.HandlerBuckets = defaultHandlerExecutionTimeBuckets + } + + m.handlerExecutionTimeSeconds, err = b.meter.Float64Histogram( + b.name("handler_execution_time_seconds"), + metric.WithUnit("seconds"), + metric.WithDescription("The total time elapsed while executing the handler function in seconds"), + metric.WithExplicitBucketBoundaries(b.HandlerBuckets...), + ) + if err != nil { + panic(errors.Wrap(err, "could not register handler execution time metric")) + } + + return m +} diff --git a/components/metrics/opentelemetry_publisher.go b/components/metrics/opentelemetry_publisher.go new file mode 100644 index 000000000..75305eb9a --- /dev/null +++ b/components/metrics/opentelemetry_publisher.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "time" + + "github.com/ThreeDotsLabs/watermill/message" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// PublisherOpenTelemetryMetricsDecorator decorates a publisher to capture OpenTelemetry metrics. +type PublisherOpenTelemetryMetricsDecorator struct { + pub message.Publisher + publisherName string + publishTimeSeconds metric.Float64Histogram +} + +// Publish updates the relevant publisher metrics and calls the wrapped publisher's Publish. +func (m PublisherOpenTelemetryMetricsDecorator) Publish(topic string, messages ...*message.Message) (err error) { + if len(messages) == 0 { + return m.pub.Publish(topic) + } + + // TODO: take ctx not only from first msg. Might require changing the signature of Publish, which is planned anyway. + ctx := messages[0].Context() + labelsMap := labelsFromCtx(ctx, publisherLabelKeys...) + if labelsMap[labelKeyPublisherName] == "" { + labelsMap[labelKeyPublisherName] = m.publisherName + } + if labelsMap[labelKeyHandlerName] == "" { + labelsMap[labelKeyHandlerName] = labelValueNoHandler + } + labels := make([]attribute.KeyValue, 0, len(labelsMap)) + for k, v := range labelsMap { + labels = append(labels, attribute.String(k, v)) + } + start := time.Now() + + defer func() { + if publishAlreadyObserved(ctx) { + // decorator idempotency when applied decorator multiple times + return + } + + if err != nil { + labels = append(labels, attribute.String(labelSuccess, "false")) + } else { + labels = append(labels, attribute.String(labelSuccess, "true")) + } + + m.publishTimeSeconds.Record( + ctx, + time.Since(start).Seconds(), + metric.WithAttributes(labels...), + ) + }() + + for _, msg := range messages { + msg.SetContext(setPublishObservedToCtx(msg.Context())) + } + + return m.pub.Publish(topic, messages...) +} + +// Close decreases the total publisher count, closes the OpenTelemetry HTTP server and calls wrapped Close. +func (m PublisherOpenTelemetryMetricsDecorator) Close() error { + return m.pub.Close() +} diff --git a/components/metrics/opentelemetry_subscriber.go b/components/metrics/opentelemetry_subscriber.go new file mode 100644 index 000000000..695d695bb --- /dev/null +++ b/components/metrics/opentelemetry_subscriber.go @@ -0,0 +1,50 @@ +package metrics + +import ( + "github.com/ThreeDotsLabs/watermill/message" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// SubscriberOpenTelemetryMetricsDecorator decorates a subscriber to capture OpenTelemetry metrics. +type SubscriberOpenTelemetryMetricsDecorator struct { + message.Subscriber + subscriberName string + subscriberMessagesReceivedTotal metric.Int64Counter +} + +func (s SubscriberOpenTelemetryMetricsDecorator) recordMetrics(msg *message.Message) { + if msg == nil { + return + } + + ctx := msg.Context() + labelsMap := labelsFromCtx(ctx, subscriberLabelKeys...) + if labelsMap[labelKeySubscriberName] == "" { + labelsMap[labelKeySubscriberName] = s.subscriberName + } + if labelsMap[labelKeyHandlerName] == "" { + labelsMap[labelKeyHandlerName] = labelValueNoHandler + } + labels := make([]attribute.KeyValue, 0, len(labelsMap)) + for k, v := range labelsMap { + labels = append(labels, attribute.String(k, v)) + } + + go func() { + if subscribeAlreadyObserved(ctx) { + // decorator idempotency when applied decorator multiple times + return + } + + select { + case <-msg.Acked(): + labels = append(labels, attribute.String(labelAcked, "acked")) + case <-msg.Nacked(): + labels = append(labels, attribute.String(labelAcked, "nacked")) + } + s.subscriberMessagesReceivedTotal.Add(ctx, 1, metric.WithAttributes(labels...)) + }() + + msg.SetContext(setSubscribeObservedToCtx(msg.Context())) +} diff --git a/components/metrics/builder.go b/components/metrics/prometheus_builder.go similarity index 100% rename from components/metrics/builder.go rename to components/metrics/prometheus_builder.go diff --git a/components/metrics/handler.go b/components/metrics/prometheus_handler.go similarity index 81% rename from components/metrics/handler.go rename to components/metrics/prometheus_handler.go index 4e3e5ea1a..83b9fb27e 100644 --- a/components/metrics/handler.go +++ b/components/metrics/prometheus_handler.go @@ -9,29 +9,6 @@ import ( "github.com/ThreeDotsLabs/watermill/message" ) -var ( - handlerLabelKeys = []string{ - labelKeyHandlerName, - labelSuccess, - } - - // defaultHandlerExecutionTimeBuckets are one order of magnitude smaller than default buckets (5ms~10s), - // because the handler execution times are typically shorter (µs~ms range). - defaultHandlerExecutionTimeBuckets = []float64{ - 0.0005, - 0.001, - 0.0025, - 0.005, - 0.01, - 0.025, - 0.05, - 0.1, - 0.25, - 0.5, - 1, - } -) - // HandlerPrometheusMetricsMiddleware is a middleware that captures Prometheus metrics. type HandlerPrometheusMetricsMiddleware struct { handlerExecutionTimeSeconds *prometheus.HistogramVec diff --git a/components/metrics/prometheus_publisher.go b/components/metrics/prometheus_publisher.go new file mode 100644 index 000000000..f217821c3 --- /dev/null +++ b/components/metrics/prometheus_publisher.go @@ -0,0 +1,58 @@ +package metrics + +import ( + "time" + + "github.com/ThreeDotsLabs/watermill/message" + "github.com/prometheus/client_golang/prometheus" +) + +// PublisherPrometheusMetricsDecorator decorates a publisher to capture Prometheus metrics. +type PublisherPrometheusMetricsDecorator struct { + pub message.Publisher + publisherName string + publishTimeSeconds *prometheus.HistogramVec +} + +// Publish updates the relevant publisher metrics and calls the wrapped publisher's Publish. +func (m PublisherPrometheusMetricsDecorator) Publish(topic string, messages ...*message.Message) (err error) { + if len(messages) == 0 { + return m.pub.Publish(topic) + } + + // TODO: take ctx not only from first msg. Might require changing the signature of Publish, which is planned anyway. + ctx := messages[0].Context() + labels := labelsFromCtx(ctx, publisherLabelKeys...) + if labels[labelKeyPublisherName] == "" { + labels[labelKeyPublisherName] = m.publisherName + } + if labels[labelKeyHandlerName] == "" { + labels[labelKeyHandlerName] = labelValueNoHandler + } + start := time.Now() + + defer func() { + if publishAlreadyObserved(ctx) { + // decorator idempotency when applied decorator multiple times + return + } + + if err != nil { + labels[labelSuccess] = "false" + } else { + labels[labelSuccess] = "true" + } + m.publishTimeSeconds.With(labels).Observe(time.Since(start).Seconds()) + }() + + for _, msg := range messages { + msg.SetContext(setPublishObservedToCtx(msg.Context())) + } + + return m.pub.Publish(topic, messages...) +} + +// Close decreases the total publisher count, closes the Prometheus HTTP server and calls wrapped Close. +func (m PublisherPrometheusMetricsDecorator) Close() error { + return m.pub.Close() +} diff --git a/components/metrics/prometheus_subscriber.go b/components/metrics/prometheus_subscriber.go new file mode 100644 index 000000000..a80e44fdb --- /dev/null +++ b/components/metrics/prometheus_subscriber.go @@ -0,0 +1,46 @@ +package metrics + +import ( + "github.com/ThreeDotsLabs/watermill/message" + "github.com/prometheus/client_golang/prometheus" +) + +// SubscriberPrometheusMetricsDecorator decorates a subscriber to capture Prometheus metrics. +type SubscriberPrometheusMetricsDecorator struct { + message.Subscriber + subscriberName string + subscriberMessagesReceivedTotal *prometheus.CounterVec + closing chan struct{} +} + +func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message) { + if msg == nil { + return + } + + ctx := msg.Context() + labels := labelsFromCtx(ctx, subscriberLabelKeys...) + if labels[labelKeySubscriberName] == "" { + labels[labelKeySubscriberName] = s.subscriberName + } + if labels[labelKeyHandlerName] == "" { + labels[labelKeyHandlerName] = labelValueNoHandler + } + + go func() { + if subscribeAlreadyObserved(ctx) { + // decorator idempotency when applied decorator multiple times + return + } + + select { + case <-msg.Acked(): + labels[labelAcked] = "acked" + case <-msg.Nacked(): + labels[labelAcked] = "nacked" + } + s.subscriberMessagesReceivedTotal.With(labels).Inc() + }() + + msg.SetContext(setSubscribeObservedToCtx(msg.Context())) +} diff --git a/components/metrics/publisher.go b/components/metrics/publisher.go index 2110429b0..b59da3b3e 100644 --- a/components/metrics/publisher.go +++ b/components/metrics/publisher.go @@ -1,12 +1,5 @@ package metrics -import ( - "time" - - "github.com/ThreeDotsLabs/watermill/message" - "github.com/prometheus/client_golang/prometheus" -) - var ( publisherLabelKeys = []string{ labelKeyHandlerName, @@ -14,53 +7,3 @@ var ( labelSuccess, } ) - -// PublisherPrometheusMetricsDecorator decorates a publisher to capture Prometheus metrics. -type PublisherPrometheusMetricsDecorator struct { - pub message.Publisher - publisherName string - publishTimeSeconds *prometheus.HistogramVec -} - -// Publish updates the relevant publisher metrics and calls the wrapped publisher's Publish. -func (m PublisherPrometheusMetricsDecorator) Publish(topic string, messages ...*message.Message) (err error) { - if len(messages) == 0 { - return m.pub.Publish(topic) - } - - // TODO: take ctx not only from first msg. Might require changing the signature of Publish, which is planned anyway. - ctx := messages[0].Context() - labels := labelsFromCtx(ctx, publisherLabelKeys...) - if labels[labelKeyPublisherName] == "" { - labels[labelKeyPublisherName] = m.publisherName - } - if labels[labelKeyHandlerName] == "" { - labels[labelKeyHandlerName] = labelValueNoHandler - } - start := time.Now() - - defer func() { - if publishAlreadyObserved(ctx) { - // decorator idempotency when applied decorator multiple times - return - } - - if err != nil { - labels[labelSuccess] = "false" - } else { - labels[labelSuccess] = "true" - } - m.publishTimeSeconds.With(labels).Observe(time.Since(start).Seconds()) - }() - - for _, msg := range messages { - msg.SetContext(setPublishObservedToCtx(msg.Context())) - } - - return m.pub.Publish(topic, messages...) -} - -// Close decreases the total publisher count, closes the Prometheus HTTP server and calls wrapped Close. -func (m PublisherPrometheusMetricsDecorator) Close() error { - return m.pub.Close() -} diff --git a/components/metrics/subscriber.go b/components/metrics/subscriber.go index c439a53a4..7c28ca618 100644 --- a/components/metrics/subscriber.go +++ b/components/metrics/subscriber.go @@ -1,53 +1,8 @@ package metrics -import ( - "github.com/ThreeDotsLabs/watermill/message" - "github.com/prometheus/client_golang/prometheus" -) - var ( subscriberLabelKeys = []string{ labelKeyHandlerName, labelKeySubscriberName, } ) - -// SubscriberPrometheusMetricsDecorator decorates a subscriber to capture Prometheus metrics. -type SubscriberPrometheusMetricsDecorator struct { - message.Subscriber - subscriberName string - subscriberMessagesReceivedTotal *prometheus.CounterVec - closing chan struct{} -} - -func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message) { - if msg == nil { - return - } - - ctx := msg.Context() - labels := labelsFromCtx(ctx, subscriberLabelKeys...) - if labels[labelKeySubscriberName] == "" { - labels[labelKeySubscriberName] = s.subscriberName - } - if labels[labelKeyHandlerName] == "" { - labels[labelKeyHandlerName] = labelValueNoHandler - } - - go func() { - if subscribeAlreadyObserved(ctx) { - // decorator idempotency when applied decorator multiple times - return - } - - select { - case <-msg.Acked(): - labels[labelAcked] = "acked" - case <-msg.Nacked(): - labels[labelAcked] = "nacked" - } - s.subscriberMessagesReceivedTotal.With(labels).Inc() - }() - - msg.SetContext(setSubscribeObservedToCtx(msg.Context())) -} diff --git a/docs/content/advanced/metrics.md b/docs/content/advanced/metrics.md index 2c5cea3c4..7011af521 100644 --- a/docs/content/advanced/metrics.md +++ b/docs/content/advanced/metrics.md @@ -12,13 +12,13 @@ We provide a default implementation using Prometheus, based on the official [Pro The `components/metrics` package exports `PrometheusMetricsBuilder`, which provides convenience functions to wrap publishers, subscribers and handlers so that they update the relevant Prometheus registry: -{{% load-snippet-partial file="src-link/components/metrics/builder.go" first_line_contains="// PrometheusMetricsBuilder" last_line_contains="func (b PrometheusMetricsBuilder)" %}} +{{% load-snippet-partial file="src-link/components/metrics/prometheus_builder.go" first_line_contains="// PrometheusMetricsBuilder" last_line_contains="func (b PrometheusMetricsBuilder)" %}} ## Wrapping publishers, subscribers and handlers If you are using Watermill's [router](/docs/messages-router) (which is recommended in most cases), you can use a single convenience function `AddPrometheusRouterMetrics` to ensure that all the handlers added to this router are wrapped to update the Prometheus registry, together with their publishers and subscribers: -{{% load-snippet-partial file="src-link/components/metrics/builder.go" first_line_contains="// AddPrometheusRouterMetrics" last_line_contains="AddMiddleware" padding_after="1" %}} +{{% load-snippet-partial file="src-link/components/metrics/prometheus_builder.go" first_line_contains="// AddPrometheusRouterMetrics" last_line_contains="AddMiddleware" padding_after="1" %}} Example use of `AddPrometheusRouterMetrics`: diff --git a/go.mod b/go.mod index 12f7b9eba..852f7e5b4 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/prometheus/client_golang v1.20.2 github.com/sony/gobreaker v1.0.0 github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/otel v1.29.0 + go.opentelemetry.io/otel/metric v1.29.0 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index 87b7d8a83..f9f05d0cf 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -57,6 +61,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=