Remove global state to try to fix flake (#7291)

## Summary of changes

Remove usages of `Tracer.Instance` to try to stop flake

## Reason for change

I've seen flake in these unit tests: e.g.

```
InjectTraceIntoData_WithJsonString_AddsTraceContext from Datadog.Trace.ClrProfiler.Managed.Tests.AutoInstrumentation.AWS.Kinesis.ContextPropagationTests (Datadog)
The given key 'dd-pathway-ctx-base64' was not present in the dictionary.
```

My strong suspicion is that that flake is due to the fact that the
tested classes are using `Tracer.Instance` state, which is global, and
so state is leaking between tests.

Also, the tracers weren't being disposed, so we were likely leaking
threads.

## Implementation details

- Inject a `Tracer` instance instead of grabbing the global one
- Use `TracerHelper.Create()` and dispose the returned agent

## Test coverage

Covered by existing tests

## Other details
 
I suspect the actual flake was introduced by
- #7244

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

Author: andrewlock

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/EventBridge/ContextPropagation.cs

@@ -23,7 +23,7 @@ internal static class ContextPropagation
         private const int MaxSizeBytes = 256 * 1024; // 256 KB
 
         // Loops through all entries of the EventBridge event and tries to inject Datadog context into each.
-        public static void InjectContext<TPutEventsRequest>(TPutEventsRequest request, PropagationContext context)
+        public static void InjectContext<TPutEventsRequest>(Tracer tracer, TPutEventsRequest request, PropagationContext context)
             where TPutEventsRequest : IPutEventsRequest, IDuckType
         {
             var entries = request.Entries.Value;
@@ -37,14 +37,14 @@ public static void InjectContext<TPutEventsRequest>(TPutEventsRequest request, P
                 var duckEntry = entry.DuckCast<IPutEventsRequestEntry>();
                 if (duckEntry != null)
                 {
-                    InjectHeadersIntoDetail(duckEntry, context);
+                    InjectHeadersIntoDetail(tracer, duckEntry, context);
                 }
             }
         }
 
         // Tries to add Datadog trace context under the `_datadog` key at the top level of the `detail` field.
         // `detail` is a string, so we have to manually modify it using a StringBuilder.
-        private static void InjectHeadersIntoDetail(IPutEventsRequestEntry entry, PropagationContext context)
+        private static void InjectHeadersIntoDetail(Tracer tracer, IPutEventsRequestEntry entry, PropagationContext context)
         {
             var detail = entry.Detail?.Trim() ?? "{}";
             if (!detail.EndsWith("}"))
@@ -61,7 +61,7 @@ private static void InjectHeadersIntoDetail(IPutEventsRequestEntry entry, Propag
                 detailBuilder.Append(','); // Add comma if the original detail is not empty
             }
 
-            var traceContext = BuildContextJson(context, entry.EventBusName);
+            var traceContext = BuildContextJson(tracer, context, entry.EventBusName);
             detailBuilder.Append($"\"{DatadogKey}\":{traceContext}").Append('}');
 
             // Check new detail size
@@ -77,13 +77,13 @@ private static void InjectHeadersIntoDetail(IPutEventsRequestEntry entry, Propag
         }
 
         // Builds a JSON string containing Datadog trace context
-        private static string BuildContextJson(PropagationContext context, string? eventBusName)
+        private static string BuildContextJson(Tracer tracer, PropagationContext context, string? eventBusName)
         {
             // Inject trace context
             var jsonBuilder = Util.StringBuilderCache.Acquire();
             jsonBuilder.Append('{');
 
-            Tracer.Instance.TracerManager.SpanContextPropagator.Inject(context, jsonBuilder, new StringBuilderCarrierSetter());
+            tracer.TracerManager.SpanContextPropagator.Inject(context, jsonBuilder, new StringBuilderCarrierSetter());
 
             // Inject start time and bus name
             var startTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/EventBridge/PutEventsAsyncIntegration.cs

@@ -50,7 +50,8 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutEventsRequest>(TTarge
             return CallTargetState.GetDefault();
         }
 
-        var scope = AwsEventBridgeCommon.CreateScope(Tracer.Instance, Operation, SpanKind, out var tags);
+        var tracer = Tracer.Instance;
+        var scope = AwsEventBridgeCommon.CreateScope(tracer, Operation, SpanKind, out var tags);
         if (tags is not null)
         {
             var busName = AwsEventBridgeCommon.GetBusName(request.Entries.Value);
@@ -63,7 +64,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutEventsRequest>(TTarge
         }
 
         var context = new PropagationContext(scope?.Span.Context, Baggage.Current);
-        ContextPropagation.InjectContext(request, context);
+        ContextPropagation.InjectContext(tracer, request, context);
 
         return new CallTargetState(scope);
     }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/EventBridge/PutEventsIntegration.cs

@@ -47,7 +47,8 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutEventsRequest>(TTarge
             return CallTargetState.GetDefault();
         }
 
-        var scope = AwsEventBridgeCommon.CreateScope(Tracer.Instance, Operation, SpanKind, out var tags);
+        var tracer = Tracer.Instance;
+        var scope = AwsEventBridgeCommon.CreateScope(tracer, Operation, SpanKind, out var tags);
         if (tags is not null)
         {
             var busName = AwsEventBridgeCommon.GetBusName(request.Entries.Value);
@@ -60,7 +61,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutEventsRequest>(TTarge
         }
 
         var context = new PropagationContext(scope?.Span.Context, Baggage.Current);
-        ContextPropagation.InjectContext(request, context);
+        ContextPropagation.InjectContext(tracer, request, context);
 
         return new CallTargetState(scope);
     }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/ContextPropagation.cs

@@ -24,7 +24,7 @@ internal static class ContextPropagation
         private const int MaxDsmHeaderSize = 34;
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ContextPropagation));
 
-        public static void InjectTraceIntoRecords<TRecordsRequest>(TRecordsRequest request, Scope? scope, string? streamName)
+        public static void InjectTraceIntoRecords<TRecordsRequest>(Tracer tracer, TRecordsRequest request, Scope? scope, string? streamName)
             where TRecordsRequest : IContainsRecords
         {
             // request.Records is not null and has at least one element
@@ -35,11 +35,11 @@ public static void InjectTraceIntoRecords<TRecordsRequest>(TRecordsRequest reque
 
             if (request.Records[0].DuckCast<IContainsData>() is { } record)
             {
-                InjectTraceIntoData(record, scope, streamName);
+                InjectTraceIntoData(tracer, record, scope, streamName);
             }
         }
 
-        public static void InjectTraceIntoData<TRecordRequest>(TRecordRequest record, Scope? scope, string? streamName)
+        public static void InjectTraceIntoData<TRecordRequest>(Tracer tracer, TRecordRequest record, Scope? scope, string? streamName)
             where TRecordRequest : IContainsData
         {
             if (scope is null)
@@ -57,7 +57,7 @@ public static void InjectTraceIntoData<TRecordRequest>(TRecordRequest record, Sc
             var propagatedContext = new Dictionary<string, object>();
             if (scope.Span.Context != null && !string.IsNullOrEmpty(streamName))
             {
-                var dataStreamsManager = Tracer.Instance.TracerManager.DataStreamsManager;
+                var dataStreamsManager = tracer.TracerManager.DataStreamsManager;
                 if (dataStreamsManager != null && dataStreamsManager.IsEnabled)
                 {
                     var payloadSize = jsonData?.Count > 0 && record.Data != null ? record.Data.Length : 0;
@@ -89,7 +89,7 @@ public static void InjectTraceIntoData<TRecordRequest>(TRecordRequest record, Sc
 
             try
             {
-                Tracer.Instance.TracerManager.SpanContextPropagator.Inject(context, propagatedContext, default(DictionaryGetterAndSetter));
+                tracer.TracerManager.SpanContextPropagator.Inject(context, propagatedContext, default(DictionaryGetterAndSetter));
                 jsonData[KinesisKey] = propagatedContext;
 
                 var memoryStreamData = DictionaryToMemoryStream(jsonData);

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordAsyncIntegration.cs

@@ -50,13 +50,14 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordRequest>(TTarge
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             if (tags is not null)
             {
                 tags.StreamName = request.StreamName;
             }
 
-            ContextPropagation.InjectTraceIntoData(request, scope, request.StreamName);
+            ContextPropagation.InjectTraceIntoData(tracer, request, scope, request.StreamName);
 
             return new CallTargetState(scope);
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordAsyncV3_7Integration.cs

@@ -50,14 +50,15 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordRequest>(TTarge
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             var streamName = AwsKinesisCommon.GetStreamName(request);
             if (tags is not null)
             {
                 tags.StreamName = streamName;
             }
 
-            ContextPropagation.InjectTraceIntoData(request, scope, streamName);
+            ContextPropagation.InjectTraceIntoData(tracer, request, scope, streamName);
 
             return new CallTargetState(scope);
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordIntegration.cs

@@ -48,13 +48,14 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordRequest>(TTarge
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             if (tags is not null)
             {
                 tags.StreamName = request.StreamName;
             }
 
-            ContextPropagation.InjectTraceIntoData(request, scope, request.StreamName);
+            ContextPropagation.InjectTraceIntoData(tracer, request, scope, request.StreamName);
 
             return new CallTargetState(scope);
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordV3_7Integration.cs

@@ -48,14 +48,15 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordRequest>(TTarge
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             var streamName = AwsKinesisCommon.GetStreamName(request);
             if (tags is not null)
             {
                 tags.StreamName = streamName;
             }
 
-            ContextPropagation.InjectTraceIntoData(request, scope, streamName);
+            ContextPropagation.InjectTraceIntoData(tracer, request, scope, streamName);
 
             return new CallTargetState(scope);
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordsAsyncIntegration.cs

@@ -50,13 +50,14 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordsRequest>(TTarg
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             if (tags is not null)
             {
                 tags.StreamName = request.StreamName;
             }
 
-            ContextPropagation.InjectTraceIntoRecords(request, scope, request.StreamName);
+            ContextPropagation.InjectTraceIntoRecords(tracer, request, scope, request.StreamName);
 
             return new CallTargetState(scope);
         }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Kinesis/PutRecordsAsyncV3_7Integration.cs

@@ -50,14 +50,15 @@ internal static CallTargetState OnMethodBegin<TTarget, TPutRecordsRequest>(TTarg
                 return CallTargetState.GetDefault();
             }
 
-            var scope = AwsKinesisCommon.CreateScope(Tracer.Instance, Operation, SpanKinds.Producer, null, out var tags);
+            var tracer = Tracer.Instance;
+            var scope = AwsKinesisCommon.CreateScope(tracer, Operation, SpanKinds.Producer, null, out var tags);
             var streamName = AwsKinesisCommon.GetStreamName(request);
             if (tags is not null)
             {
                 tags.StreamName = streamName;
             }
 
-            ContextPropagation.InjectTraceIntoRecords(request, scope, streamName);
+            ContextPropagation.InjectTraceIntoRecords(tracer, request, scope, streamName);
 
             return new CallTargetState(scope);
         }