diff --git a/api/v1beta3/provider_types.go b/api/v1beta3/provider_types.go index b374ac86e..7a83896db 100644 --- a/api/v1beta3/provider_types.go +++ b/api/v1beta3/provider_types.go @@ -53,13 +53,14 @@ const ( DataDogProvider string = "datadog" NATSProvider string = "nats" ZulipProvider string = "zulip" + OTELProvider string = "otel" ) // ProviderSpec defines the desired state of the Provider. // +kubebuilder:validation:XValidation:rule="self.type == 'github' || self.type == 'gitlab' || self.type == 'gitea' || self.type == 'bitbucketserver' || self.type == 'bitbucket' || self.type == 'azuredevops' || !has(self.commitStatusExpr)", message="spec.commitStatusExpr is only supported for the 'github', 'gitlab', 'gitea', 'bitbucketserver', 'bitbucket', 'azuredevops' provider types" type ProviderSpec struct { // Type specifies which Provider implementation to use. - // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucketserver;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty;datadog;nats;zulip + // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucketserver;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty;datadog;nats;zulip;otel // +required Type string `json:"type"` diff --git a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml index e42cfd81b..00a581dda 100644 --- a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml +++ b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml @@ -386,6 +386,7 @@ spec: - datadog - nats - zulip + - otel type: string username: description: Username specifies the name under which events are posted. diff --git a/docs/spec/v1beta3/providers.md b/docs/spec/v1beta3/providers.md index ea09e15b9..11bd3b86f 100644 --- a/docs/spec/v1beta3/providers.md +++ b/docs/spec/v1beta3/providers.md @@ -109,6 +109,7 @@ The supported alerting providers are: | [WebEx](#webex) | `webex` | | [NATS](#nats) | `nats` | | [Zulip](#zulip) | `zulip` | +| [OTEL](#otel) | `otel` | #### Types supporting Git commit status updates @@ -1180,6 +1181,49 @@ stringData: password: F8KXuAylZOta3L5tjgVm3r1YVruUNGXu # the Zulip bot API key ``` +##### OTEL + +When `.spec.type` is set to `otel`, the controller will send distributed tracing data for +[Events](events.md#event-structure) to an OpenTelemetry (OTEL) collector endpoint specified in the [Address](#address) field. + +The controller converts Flux events into OTEL spans with proper trace relationships based on the Flux object hierarchy. Source objects (GitRepository, HelmChart, OCIRepository, Bucket) create root spans, while other objects create child spans within the same trace. Each span includes event metadata as attributes and uses the alert name and namespace as the service identifier. + +Spans are correlated using a trace ID generated from the alert UID and revision metadata (depending on the source, the revision could be oci-digest, git commit hash, etc.). Having the following format: `:`. All events from the same alert share the same trace ID, enabling end-to-end visibility across. + +This Provider type supports authentication via [Secret reference](#secret-reference), [proxy configuration](#https-proxy), and [TLS certificates](#certificate-secret-reference). + +**Note:** HelmRepository events are skipped as they are not considered primary sources for tracing. + +###### OTEL with Basic Authentication Example + +To configure `otel` provider with basic authentication, create a Secret with the +`username` and `password` fields set, and add a `otel` provider with the associated +[Secret reference](#secret-reference). + +```yaml +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: otel-provider + namespace: desired-namespace +spec: + type: otel + address: + secretRef: + name: otel-provider-creds +--- +apiVersion: v1 +kind: Secret +metadata: + name: otel-provider-creds + namespace: desired-namespace +stringData: + username: + password: +``` + + ### Address `.spec.address` is an optional field that specifies the endpoint where the events are posted. @@ -1342,6 +1386,7 @@ The following providers support client certificate authentication: | `slack` | Slack API | | `webex` | Webex messages | | `zulip` | Zulip API | +| `otel` | OpenTelemetry Traces | Support for client certificate authentication is being expanded to additional providers over time. diff --git a/go.mod b/go.mod index 56e1fcf19..b5087202f 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,11 @@ require ( github.com/slok/go-http-metrics v0.13.0 github.com/spf13/pflag v1.0.7 gitlab.com/gitlab-org/api/client-go v0.137.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 golang.org/x/oauth2 v0.30.0 golang.org/x/text v0.28.0 google.golang.org/api v0.248.0 @@ -76,6 +81,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudevents/sdk-go/v2 v2.15.2 // indirect @@ -131,6 +137,7 @@ require ( github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -174,11 +181,10 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect @@ -193,9 +199,9 @@ require ( golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect - google.golang.org/grpc v1.74.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 32c434810..a6999ee2f 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,8 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.16.0 h1:B91r9bHtXp/+XRgS5aZm6ZzTdz github.com/bradleyfalzon/ghinstallation/v2 v2.16.0/go.mod h1:OeVe5ggFzoBnmgitZe/A+BqGOnv1DvU/0uiLQi1wutM= github.com/cdevents/sdk-go v0.4.1 h1:Cr/iH/I51Z+slxKRx9AV7stn6hr2pjRHQ5wpPJhRLTU= github.com/cdevents/sdk-go v0.4.1/go.mod h1:3IhWLoY4vsyUEzv7XJbyr0BRQ0KPgvNx+wiD2hQGFNU= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -164,8 +166,8 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -280,6 +282,8 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -432,20 +436,26 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -531,6 +541,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.248.0 h1:hUotakSkcwGdYUqzCRc5yGYsg4wXxpkKlW5ryVqvC1Y= google.golang.org/api v0.248.0/go.mod h1:yAFUAF56Li7IuIQbTFoLwXTCI6XCFKueOlS7S9e4F9k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -540,17 +552,17 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/notifier/factory.go b/internal/notifier/factory.go index 6567cfa83..8be742762 100644 --- a/internal/notifier/factory.go +++ b/internal/notifier/factory.go @@ -61,6 +61,7 @@ var ( apiv1.BitbucketProvider: bitbucketNotifierFunc, apiv1.AzureDevOpsProvider: azureDevOpsNotifierFunc, apiv1.ZulipProvider: zulipNotifierFunc, + apiv1.OTELProvider: otelNotifierFunc, } ) @@ -360,3 +361,10 @@ func azureDevOpsNotifierFunc(opts notifierOptions) (Interface, error) { func zulipNotifierFunc(opts notifierOptions) (Interface, error) { return NewZulip(opts.URL, opts.Channel, opts.ProxyURL, opts.TLSConfig, opts.Username, opts.Password) } + +func otelNotifierFunc(opts notifierOptions) (Interface, error) { + if opts.Token == "" && opts.Password != "" { + opts.Token = opts.Password + } + return NewOTLPTracer(opts.Context, opts.URL, opts.ProxyURL, opts.Headers, opts.TLSConfig, opts.Username, opts.Token) +} diff --git a/internal/notifier/otel.go b/internal/notifier/otel.go new file mode 100644 index 000000000..769d7b80c --- /dev/null +++ b/internal/notifier/otel.go @@ -0,0 +1,258 @@ +/* +Copyright 2025 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "slices" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "go.opentelemetry.io/otel/trace" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" + + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" + + apiv1beta3 "github.com/fluxcd/notification-controller/api/v1beta3" +) + +// Context key +type alertMetadataContextKey struct{} + +func WithAlertMetadata(ctx context.Context, metadata metav1.ObjectMeta) context.Context { + return context.WithValue(ctx, alertMetadataContextKey{}, metadata) +} + +func GetAlertMetadata(ctx context.Context) (metav1.ObjectMeta, bool) { + metadata, ok := ctx.Value(alertMetadataContextKey{}).(metav1.ObjectMeta) + return metadata, ok +} + +type OTLPTracer struct { + tracerExporter *otlptrace.Exporter +} + +func NewOTLPTracer(ctx context.Context, urlStr string, proxyURL string, headers map[string]string, tlsConfig *tls.Config, username string, password string) (*OTLPTracer, error) { + // Set up OTLP exporter options + httpOptions := []otlptracehttp.Option{ + otlptracehttp.WithEndpointURL(urlStr), + } + + // Add headers if available + if len(headers) > 0 { + // Add authentication header, if it doesn't exist yet + if headers["Authorization"] == "" { + // If username is not set, password is considered as token + if username == "" { + headers["Authorization"] = "Bearer " + password + } else if password != "" { + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + headers["Authorization"] = "Basic " + auth + } + } + httpOptions = append(httpOptions, otlptracehttp.WithHeaders(headers)) + } + + // Add TLS config if available + if tlsConfig != nil { + httpOptions = append(httpOptions, otlptracehttp.WithTLSClientConfig(tlsConfig)) + } + + // Add proxy if available + if proxyURL != "" { + proxyURLparsed, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("failed to proxy URL - %s: %w", proxyURL, err) + } else { + if username != "" && password != "" { + proxyURLparsed.User = url.UserPassword(username, password) + } + httpOptions = append(httpOptions, otlptracehttp.WithProxy(func(*http.Request) (*url.URL, error) { + return proxyURLparsed, nil + })) + } + } + + exporter, err := otlptracehttp.New(ctx, httpOptions...) + if err != nil { + return nil, err + } + + log.FromContext(ctx).V(1).Info("Successfully created OTEL tracerExporter") + return &OTLPTracer{ + tracerExporter: exporter, + }, nil +} + +// Post implements the notifier.Interface +func (t *OTLPTracer) Post(ctx context.Context, event eventv1.Event) error { + // Skip Git commit status update event. + if event.HasMetadata(eventv1.MetaCommitStatusKey, eventv1.MetaCommitStatusUpdateValue) { + return nil + } + + logger := log.FromContext(ctx).V(1).WithValues( + "event", event.Reason, + "object", fmt.Sprintf("%s/%s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Namespace, event.InvolvedObject.Name), + "severity", event.Severity, + ) + logger.Info("OTEL Post function called", "event", event.Reason) + + alert, ok := GetAlertMetadata(ctx) + if !ok { + return fmt.Errorf("alert metadata not found in context") + } + + // Extract revision from event metadata + revision := getRevision(event.Metadata) + + // Create TraceProvider + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(sdktrace.NewSimpleSpanProcessor(t.tracerExporter)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(fmt.Sprintf("%s/%s", alert.GetNamespace(), alert.GetName())), + semconv.ServiceNamespace(alert.GetNamespace()), + semconv.ServiceInstanceID(string(alert.GetUID())), + ), + ), + ) + + // Generate traceID + logger.V(1).Info("Generating trace IDs", "alertUID", string(alert.UID), "revision", revision) + var traceID trace.TraceID + traceIDStr := generateID(string(alert.UID), revision) + copy(traceID[:], traceIDStr[:16]) + + // Determine span relationship based on Flux object hierarchy + var spanCtx context.Context = createSpanContext(ctx, event, traceID) + + // Skip if it's HelmRepository kind object (no considered as main source for tracing) + if event.InvolvedObject.Kind != "HelmRepository" { + logger.Info("Processing OTEL notification", "alert", alert.Name) + + } else { + logger.Info("OTEL notification skipped", "alert", alert.Name) + return nil + } + + // Create single span with proper attributes + span := processSpan(tp, spanCtx, event) + // Set status based on event severity + if event.Severity == eventv1.EventSeverityError { + span.SetStatus(codes.Error, event.Message) + } else { + span.SetStatus(codes.Ok, event.Message) + } + + defer span.End() + + serviceName := fmt.Sprintf("%s: %s/%s", apiv1beta3.AlertKind, alert.Namespace, alert.Name) + logger.Info("Successfully sent trace to OTLP endpoint", + "alert", serviceName, + ) + + return nil +} + +func createSpanContext(ctx context.Context, event eventv1.Event, traceID trace.TraceID) context.Context { + kind := event.InvolvedObject.Kind + + spanContext := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, + TraceFlags: trace.FlagsSampled, + Remote: true, + }) + // Root spans: Sources that start the deployment flow + if isSource(kind) { + return trace.ContextWithSpanContext(context.Background(), + spanContext.WithTraceFlags(spanContext.TraceFlags())) + } + + // Child spans: Everything else inherits from the same trace + return trace.ContextWithSpanContext(ctx, + spanContext.WithTraceFlags(spanContext.TraceFlags())) +} + +func processSpan(tracerProvider *sdktrace.TracerProvider, ctx context.Context, event eventv1.Event) trace.Span { + // Build span attributes including metadata + eventAttrs := []attribute.KeyValue{ + attribute.String("object.uid", string(event.InvolvedObject.UID)), + attribute.String("object.kind", event.InvolvedObject.Kind), + attribute.String("object.name", event.InvolvedObject.Name), + attribute.String("object.namespace", event.InvolvedObject.Namespace), + } + + // Add event metadata as span attributes + for k, v := range event.Metadata { + eventAttrs = append(eventAttrs, attribute.String(k, v)) + } + + // Create tracer and start tracing + spanName := fmt.Sprintf("%s: %s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Namespace, event.InvolvedObject.Name) + tracer := tracerProvider.Tracer("flux:notification-controller") + _, span := tracer.Start(ctx, spanName, + trace.WithAttributes(eventAttrs...), + trace.WithTimestamp(event.Timestamp.Time)) + + return span +} + +// Build the revision ID based on the event metadata +func getRevision(eventMetadata map[string]string) string { + var revision string = "unknown" + + // OCIRepositories does populate the following metadata + // which it's the same revision as some other sources + // @ -> @: + ociDigest, hasOCI := eventMetadata["oci-digest"] + appVersion, hasApp := eventMetadata["app-version"] + + if hasOCI && hasApp { + revision = appVersion + "@" + ociDigest + } else if rev, hasRev := eventMetadata["revision"]; hasRev { + revision = rev + } + + return revision +} + +// Generate IDs based on: UID + revision +func generateID(UID string, revision string) []byte { + input := fmt.Sprintf("%s:%s", UID, revision) + hash := sha256.Sum256([]byte(input)) + return hash[:] +} + +// Discriminates if an object kind is a source +func isSource(kind string) bool { + sourceKinds := []string{"GitRepository", "HelmChart", "OCIRepository", "Bucket"} + return slices.Contains(sourceKinds, kind) +} diff --git a/internal/notifier/otel_test.go b/internal/notifier/otel_test.go new file mode 100644 index 000000000..9b433b490 --- /dev/null +++ b/internal/notifier/otel_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2025 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fluxcd/pkg/apis/event/v1beta1" +) + +func TestOTEL_Post(t *testing.T) { + g := NewWithT(t) + var receivedRequests []*http.Request + var receivedBodies [][]byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedRequests = append(receivedRequests, r) + body := make([]byte, r.ContentLength) + r.Body.Read(body) + receivedBodies = append(receivedBodies, body) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + tests := []struct { + name string + event func() v1beta1.Event + }{ + { + name: "test event", + event: func() v1beta1.Event { + e := testEvent() + // Mocking the data provided by alert.eventMetadata + e.Metadata["cluster"] = "my-cluster" + e.Metadata["region"] = "us-east-2" + e.Metadata["env"] = "prod" + return e + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + alertMetadata := &metav1.ObjectMeta{ + Name: "test-alert", + Namespace: "test-namespace", + UID: "test-alert-uid", + } + ctx := WithAlertMetadata(context.Background(), *alertMetadata) + + otelTrace, err := NewOTLPTracer(ctx, ts.URL, "", nil, nil, "", "") + g.Expect(err).ToNot(HaveOccurred()) + + err = otelTrace.Post(ctx, tt.event()) + g.Expect(err).ToNot(HaveOccurred()) + + g.Eventually(func() int { + return len(receivedRequests) + }, time.Second*5, time.Millisecond*200).Should(BeNumerically(">", 0)) + + // Check the request + g.Expect(receivedRequests).To(HaveLen(1)) + req := receivedRequests[0] + g.Expect(req.Method).To(Equal("POST")) + g.Expect(req.Header.Get("Content-Type")).To(ContainSubstring("application/x-protobuf")) + g.Expect(receivedBodies[0]).ToNot(BeEmpty()) + + // Validate OTLP content contains expected span data + body := string(receivedBodies[0]) + g.Expect(body).To(ContainSubstring(tt.event().InvolvedObject.Name)) + g.Expect(body).To(ContainSubstring(tt.event().InvolvedObject.Kind)) + g.Expect(body).To(ContainSubstring(tt.event().InvolvedObject.Namespace)) + // Check for the actual transformed attributes: + g.Expect(body).To(ContainSubstring("my-cluster")) // cluster value + g.Expect(body).To(ContainSubstring("us-east-2")) // region value + g.Expect(body).To(ContainSubstring("prod")) // env value + }) + } +} diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go index be506f7e2..d41b291eb 100644 --- a/internal/server/event_handlers.go +++ b/internal/server/event_handlers.go @@ -217,6 +217,7 @@ func (s *EventServer) dispatchNotification(ctx context.Context, event *eventv1.E go func(n notifier.Interface, e eventv1.Event) { pctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + pctx = notifier.WithAlertMetadata(pctx, alert.ObjectMeta) if err := n.Post(pctx, e); err != nil { maskedErrStr, maskErr := masktoken.MaskTokenFromString(err.Error(), token) if maskErr != nil {