wire up otel tracing to track2 Azure SDK

Author: nojnhuh

azure/defaults.go

@@ -17,6 +17,7 @@ limitations under the License.
 package azure
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -26,6 +27,8 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
+	"github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel"
+	"sigs.k8s.io/cluster-api-provider-azure/pkg/ot"
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
 	"sigs.k8s.io/cluster-api-provider-azure/version"
 )
@@ -378,6 +381,12 @@ func ARMClientOptions(azureEnvironment string, extraPolicies ...policy.Policy) (
 	opts.PerCallPolicies = append(opts.PerCallPolicies, extraPolicies...)
 	opts.Retry.MaxRetries = -1 // Less than zero means one try and no retries.
 
+	otelTP, err := ot.OTLPTracerProvider(context.TODO())
+	if err != nil {
+		return nil, err
+	}
+	opts.TracingProvider = azotel.NewTracingProvider(otelTP, nil)
+
 	return opts, nil
 }
 

azure/scope/identity.go

@@ -23,11 +23,13 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/pkg/ot"
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -81,6 +83,12 @@ func (p *AzureCredentialsProvider) GetTokenCredential(ctx context.Context, resou
 	var authErr error
 	var cred azcore.TokenCredential
 
+	otelTP, err := ot.OTLPTracerProvider(ctx)
+	if err != nil {
+		return nil, err
+	}
+	tracingProvider := azotel.NewTracingProvider(otelTP, nil)
+
 	switch p.Identity.Spec.Type {
 	case infrav1.WorkloadIdentity:
 		azwiCredOptions, err := NewWorkloadIdentityCredentialOptions().
@@ -90,6 +98,7 @@ func (p *AzureCredentialsProvider) GetTokenCredential(ctx context.Context, resou
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to setup azwi options for identity %s", p.Identity.Name)
 		}
+		azwiCredOptions.ClientOptions.TracingProvider = tracingProvider
 		cred, authErr = NewWorkloadIdentityCredential(azwiCredOptions)
 
 	case infrav1.ManualServicePrincipal:
@@ -102,6 +111,7 @@ func (p *AzureCredentialsProvider) GetTokenCredential(ctx context.Context, resou
 		}
 		options := azidentity.ClientSecretCredentialOptions{
 			ClientOptions: azcore.ClientOptions{
+				TracingProvider: tracingProvider,
 				Cloud: cloud.Configuration{
 					ActiveDirectoryAuthorityHost: activeDirectoryEndpoint,
 					Services: map[cloud.ServiceName]cloud.ServiceConfiguration{
@@ -124,10 +134,17 @@ func (p *AzureCredentialsProvider) GetTokenCredential(ctx context.Context, resou
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse certificate data")
 		}
-		cred, authErr = azidentity.NewClientCertificateCredential(p.GetTenantID(), p.Identity.Spec.ClientID, certs, key, nil)
+		cred, authErr = azidentity.NewClientCertificateCredential(p.GetTenantID(), p.Identity.Spec.ClientID, certs, key, &azidentity.ClientCertificateCredentialOptions{
+			ClientOptions: azcore.ClientOptions{
+				TracingProvider: tracingProvider,
+			},
+		})
 
 	case infrav1.UserAssignedMSI:
 		options := azidentity.ManagedIdentityCredentialOptions{
+			ClientOptions: azcore.ClientOptions{
+				TracingProvider: tracingProvider,
+			},
 			ID: azidentity.ClientID(p.Identity.Spec.ClientID),
 		}
 		cred, authErr = azidentity.NewManagedIdentityCredential(&options)

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcehealth/armresourcehealth v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0
 	github.com/Azure/azure-service-operator/v2 v2.8.0
 	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.13

go.sum

@@ -77,6 +77,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 h1:UrGzkHueDwAWDdjQxC+QaXHd4tVCkISYE9j7fSSXF8k=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0/go.mod h1:qskvSQeW+cxEE2bcKYyKimB1/KiQ9xpJ99bcHY0BX6c=
+github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 h1:RTTsXUJWn0jumeX62Mb153wYXykqnrzYBYDeHp0kiuk=
+github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0/go.mod h1:k4MMjrPHIEK+umaMGk1GNLgjEybJZ9mHSRDZ+sDFv3Y=
 github.com/Azure/azure-service-operator/v2 v2.8.0 h1:BcyB8LvRmtgVIIUaXwWIJz5eHvknyno0qq5LkDuvM/s=
 github.com/Azure/azure-service-operator/v2 v2.8.0/go.mod h1:ezbJS56PcORFFqLV8XZmM9xZ12m6aGAkg353fQhWD/8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
@@ -501,6 +503,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0=
 go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
 go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
+go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA=
+go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s=

pkg/ot/traces.go

@@ -34,7 +34,7 @@ import (
 
 // RegisterTracing enables code tracing via OpenTelemetry.
 func RegisterTracing(ctx context.Context, log logr.Logger) error {
-	tp, err := otlpTracerProvider(ctx, "opentelemetry-collector:4317")
+	tp, err := OTLPTracerProvider(ctx)
 	if err != nil {
 		return err
 	}
@@ -53,8 +53,8 @@ func RegisterTracing(ctx context.Context, log logr.Logger) error {
 	return nil
 }
 
-// otlpTracerProvider initializes an OTLP exporter and configures the corresponding tracer provider.
-func otlpTracerProvider(ctx context.Context, url string) (*sdktrace.TracerProvider, error) {
+// OTLPTracerProvider initializes an OTLP exporter and configures the corresponding tracer provider.
+func OTLPTracerProvider(ctx context.Context) (*sdktrace.TracerProvider, error) {
 	res, err := resource.New(ctx,
 		resource.WithAttributes(
 			semconv.ServiceNameKey.String("capz"),
@@ -69,7 +69,7 @@ func otlpTracerProvider(ctx context.Context, url string) (*sdktrace.TracerProvid
 
 	traceExporter, err := otlptracegrpc.New(ctx,
 		otlptracegrpc.WithInsecure(),
-		otlptracegrpc.WithEndpoint(url),
+		otlptracegrpc.WithEndpoint("opentelemetry-collector:4317"),
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create otlp trace exporter")