[Test Optimization] Performance Improvements

Datadog.Trace.TestOptimization.slnf

@@ -11,8 +11,8 @@
       "tracer\\src\\Datadog.Trace.Manual\\Datadog.Trace.Manual.csproj",
       "tracer\\src\\Datadog.Trace.MSBuild\\Datadog.Trace.MSBuild.csproj",
       "tracer\\src\\Datadog.Trace.SourceGenerators\\Datadog.Trace.SourceGenerators.csproj",
-      "tracer\\src\\Datadog.Trace.Tools.Analyzers\\Datadog.Trace.Tools.Analyzers.csproj",
       "tracer\\src\\Datadog.Trace.Tools.Analyzers.CodeFixes\\Datadog.Trace.Tools.Analyzers.CodeFixes.csproj",
+      "tracer\\src\\Datadog.Trace.Tools.Analyzers\\Datadog.Trace.Tools.Analyzers.csproj",
       "tracer\\src\\Datadog.Trace.Tools.Runner\\Datadog.Trace.Tools.Runner.csproj",
       "tracer\\src\\Datadog.Trace.Tools.Shared\\Datadog.Trace.Tools.Shared.csproj",
       "tracer\\src\\Datadog.Trace\\Datadog.Trace.csproj",

tracer/src/Datadog.Trace.Tools.Runner/CiUtils.cs

@@ -134,13 +134,6 @@ public static async Task<InitResults> InitializeCiCommandsAsync(
 
             // Change the .git search folder to the CurrentDirectory or WorkingFolder
             var ciValues = testOptimization.CIValues;
-            ciValues.GitSearchFolder = Environment.CurrentDirectory;
-            if (string.IsNullOrEmpty(ciValues.WorkspacePath))
-            {
-                // In case we cannot get the WorkspacePath we fallback to the default configuration.
-                ciValues.GitSearchFolder = null;
-            }
-
             var client = TestOptimizationClient.Create(ciValues.WorkspacePath ?? Environment.CurrentDirectory, testOptimization);
             if (testOptimizationSettings.GitUploadEnabled != false || testOptimizationSettings.IntelligentTestRunnerEnabled)
             {

tracer/src/Datadog.Trace.Tools.Runner/RunCiCommand.cs

@@ -84,8 +84,11 @@ private async Task ExecuteAsync(InvocationContext context)
                 processInfo.ArgumentList.Add(arg);
             }
 
+            Log.Debug("RunCiCommand.FileName: '{FileName}'", processInfo.FileName);
+            Log.Debug("RunCiCommand.Arguments: '{Arguments}'", string.Join(" ", processInfo.ArgumentList));
+            Log.Debug("RunCiCommand.WorkingDirectory: '{WorkingDirectory}'", processInfo.WorkingDirectory);
             var exitCode = Utils.RunProcess(processInfo, _applicationContext.TokenSource.Token);
-            Log.Debug<int>("RunCiCommand: Finished with exit code: {Value}", exitCode);
+            Log.Debug<int>("RunCiCommand.ExitCode: {Value}", exitCode);
 
             if (!initResults.TestSkippingEnabled)
             {

tracer/src/Datadog.Trace/Ci/Agent/CIVisibilityProtocolWriter.cs

@@ -98,7 +98,7 @@ public CIVisibilityProtocolWriter(
             _buffersArray[i].SetFlushTask(tskFlush);
         }
 
-        Log.Information<int>("CIVisibilityProtocolWriter Initialized with concurrency level of: {ConcurrencyLevel}", concurrencyLevel);
+        Log.Debug<int>("CIVisibilityProtocolWriter Initialized with concurrency level of: {ConcurrencyLevel}", concurrencyLevel);
     }
 
     public void WriteEvent(IEvent @event)
@@ -349,7 +349,7 @@ internal void SetFlushTask(Task flushTask)
 
         public Task FlushCiTestCycleBufferWhenTimeElapsedAsync(int batchInterval)
         {
-            return CiTestCycleBufferWatch.ElapsedMilliseconds >= batchInterval ?
+            return CiTestCycleBufferWatch.GetElapsedMilliseconds() >= batchInterval ?
                        FlushCiTestCycleBufferAsync() : Task.CompletedTask;
         }
 
@@ -367,7 +367,7 @@ async Task InternalFlushCiTestCycleBufferAsync()
 
         public Task FlushCiCodeCoverageBufferWhenTimeElapsedAsync(int batchInterval)
         {
-            return CiCodeCoverageBufferWatch.ElapsedMilliseconds >= batchInterval ?
+            return CiCodeCoverageBufferWatch.GetElapsedMilliseconds() >= batchInterval ?
                        FlushCiCodeCoverageBufferAsync() : Task.CompletedTask;
         }
 

tracer/src/Datadog.Trace/Ci/Agent/CIWriterHttpSender.cs

@@ -16,6 +16,7 @@
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Telemetry;
+using Datadog.Trace.Util;
 using Datadog.Trace.Util.Http;
 using Datadog.Trace.Vendors.Serilog.Events;
 
@@ -34,7 +35,7 @@ public CIWriterHttpSender(IApiRequestFactory apiRequestFactory)
     {
         _apiRequestFactory = apiRequestFactory;
         _isDebugEnabled = GlobalSettings.Instance.DebugEnabled;
-        Log.Information("CIWriterHttpSender Initialized.");
+        Log.Debug("CIWriterHttpSender Initialized.");
     }
 
     public Task SendPayloadAsync(EventPlatformPayload payload)
@@ -146,7 +147,7 @@ private async Task SendPayloadAsync<T>(EventPlatformPayload payload, Func<IApiRe
             }
             finally
             {
-                TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointPayloadRequestsMs(payload.TelemetryEndpoint, sw.Elapsed.TotalMilliseconds);
+                TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointPayloadRequestsMs(payload.TelemetryEndpoint, sw.GetElapsedMilliseconds());
                 if (TelemetryHelper.GetErrorTypeFromStatusCode(statusCode) is { } errorType)
                 {
                     TelemetryFactory.Metrics.RecordCountCIVisibilityEndpointPayloadRequestsErrors(payload.TelemetryEndpoint, errorType);

tracer/src/Datadog.Trace/Ci/Agent/Payloads/CICodeCoveragePayload.cs

@@ -15,6 +15,7 @@
 using Datadog.Trace.Ci.Coverage.Models.Tests;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Telemetry.Metrics;
+using Datadog.Trace.Util;
 using Datadog.Trace.Vendors.MessagePack;
 
 namespace Datadog.Trace.Ci.Agent.Payloads;
@@ -69,7 +70,7 @@ public override bool TryProcessEvent(IEvent @event)
             TestCoverageEventsCount++;
         }
 
-        TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointEventsSerializationMs(TelemetryEndpoint, _serializationWatch.Elapsed.TotalMilliseconds);
+        TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointEventsSerializationMs(TelemetryEndpoint, _serializationWatch.GetElapsedMilliseconds());
         return success;
     }
 

tracer/src/Datadog.Trace/Ci/Agent/Payloads/CIVisibilityProtocolPayload.cs

@@ -10,6 +10,7 @@
 using Datadog.Trace.Ci.Configuration;
 using Datadog.Trace.Ci.EventModel;
 using Datadog.Trace.Telemetry;
+using Datadog.Trace.Util;
 using Datadog.Trace.Vendors.MessagePack;
 
 namespace Datadog.Trace.Ci.Agent.Payloads;
@@ -74,7 +75,7 @@ public override bool TryProcessEvent(IEvent @event)
             }
         }
 
-        TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointEventsSerializationMs(TelemetryEndpoint, _serializationWatch.Elapsed.TotalMilliseconds);
+        TelemetryFactory.Metrics.RecordDistributionCIVisibilityEndpointEventsSerializationMs(TelemetryEndpoint, _serializationWatch.GetElapsedMilliseconds());
         return success;
     }
 

tracer/src/Datadog.Trace/Ci/CiEnvironment/CIEnvironmentValuesGenerics.cs

@@ -13,6 +13,8 @@
 using System.Text.RegularExpressions;
 using Datadog.Trace.Ci.Tags;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Util;
+using Datadog.Trace.Vendors.Serilog.Events;
 
 namespace Datadog.Trace.Ci.CiEnvironment;
 
@@ -26,6 +28,7 @@ internal abstract class CIEnvironmentValues<TValueProvider>(TValueProvider value
 
     internal static CIEnvironmentValues Create(TValueProvider valueProvider)
     {
+        Log.Debug("CIEnvironmentValues: Creating instance.");
         if (!string.IsNullOrEmpty(valueProvider.GetValue(Constants.Travis)))
         {
             return new TravisEnvironmentValues<TValueProvider>(valueProvider);
@@ -107,6 +110,35 @@ internal static CIEnvironmentValues Create(TValueProvider valueProvider)
 
     protected override void Setup(IGitInfo gitInfo)
     {
+        if (gitInfo.Errors.Count > 0)
+        {
+            var sb = StringBuilderCache.Acquire();
+            sb.AppendLine();
+            foreach (var err in gitInfo.Errors)
+            {
+                sb.AppendLine(" Error: " + err);
+            }
+
+            Log.Warning("CIEnvironmentValues: Errors detected in the local gitInfo: {Errors}", StringBuilderCache.GetStringAndRelease(sb));
+        }
+
+        if (Log.IsEnabled(LogEventLevel.Debug))
+        {
+            var sb = StringBuilderCache.Acquire();
+            sb.AppendLine();
+            var values = ValueProvider.GetValues();
+            foreach (var field in typeof(CIEnvironmentValues.Constants).GetFields())
+            {
+                var fieldName = field.GetValue(null) as string;
+                if (!StringUtil.IsNullOrEmpty(fieldName) && values.TryGetValue<string>(fieldName, out var value))
+                {
+                    sb.AppendFormat("\t{0}={1}{2}", fieldName, value == string.Empty ? "(empty)" : $"\"{value}\"", Environment.NewLine);
+                }
+            }
+
+            Log.Debug("CIEnvironmentValues: Values detected:{Values}", StringBuilderCache.GetStringAndRelease(sb));
+        }
+
         OnInitialize(gitInfo);
 
         // **********
@@ -159,7 +191,7 @@ protected override void Setup(IGitInfo gitInfo)
         }
         else
         {
-            Log.Warning("Git commit in .git folder is different from the one in the environment variables. [{GitCommit} != {EnvVarCommit}]", gitInfo.Commit, Commit);
+            Log.Warning("CIEnvironmentValues: Git commit in .git folder is different from the one in the environment variables. [{GitCommit} != {EnvVarCommit}]", gitInfo.Commit, Commit);
         }
 
         // **********
@@ -181,7 +213,7 @@ protected override void Setup(IGitInfo gitInfo)
             }
             else
             {
-                Log.Warning("Error fetching data for git commit '{HeadCommit}'", HeadCommit);
+                Log.Warning("CIEnvironmentValues: Error fetching data for git commit '{HeadCommit}'", HeadCommit);
             }
         }
 
@@ -209,11 +241,11 @@ protected override void Setup(IGitInfo gitInfo)
                     {
                         if (string.IsNullOrEmpty(defaultValue))
                         {
-                            Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an empty value, and the Git repository could not be automatically extracted");
+                            Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an empty value, and the Git repository could not be automatically extracted");
                         }
                         else
                         {
-                            Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an empty value, defaulting to '{Default}'", defaultValue);
+                            Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an empty value, defaulting to '{Default}'", defaultValue);
                         }
 
                         return false;
@@ -223,11 +255,11 @@ protected override void Setup(IGitInfo gitInfo)
                     {
                         if (string.IsNullOrEmpty(defaultValue))
                         {
-                            Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), and the Git repository could not be automatically extracted", value);
+                            Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), and the Git repository could not be automatically extracted", value);
                         }
                         else
                         {
-                            Log.ErrorSkipTelemetry("DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), defaulting to '{Default}'", value, defaultValue);
+                            Log.ErrorSkipTelemetry("CIEnvironmentValues: DD_GIT_REPOSITORY_URL is set with an invalid value ('{Value}'), defaulting to '{Default}'", value, defaultValue);
                         }
 
                         return false;
@@ -239,7 +271,7 @@ protected override void Setup(IGitInfo gitInfo)
 
                 if (string.IsNullOrEmpty(defaultValue))
                 {
-                    Log.Warning("The Git repository couldn't be automatically extracted.");
+                    Log.Warning("CIEnvironmentValues: The Git repository couldn't be automatically extracted.");
                 }
 
                 // If not set use the default value
@@ -257,11 +289,11 @@ protected override void Setup(IGitInfo gitInfo)
                     {
                         if (string.IsNullOrEmpty(defaultValue))
                         {
-                            Log.Error("DD_GIT_COMMIT_SHA must be a full-length git SHA, and the The Git commit sha couldn't be automatically extracted.");
+                            Log.Error("CIEnvironmentValues: DD_GIT_COMMIT_SHA must be a full-length git SHA ({Value}), and the The Git commit sha couldn't be automatically extracted.", value);
                         }
                         else
                         {
-                            Log.Error("DD_GIT_COMMIT_SHA must be a full-length git SHA, defaulting to '{Default}", defaultValue);
+                            Log.Error("CIEnvironmentValues: DD_GIT_COMMIT_SHA must be a full-length git SHA ({Value}), defaulting to '{Default}", value, defaultValue);
                         }
 
                         return false;
@@ -273,7 +305,7 @@ protected override void Setup(IGitInfo gitInfo)
 
                 if (string.IsNullOrEmpty(defaultValue))
                 {
-                    Log.Warning("The Git commit sha couldn't be automatically extracted.");
+                    Log.Warning("CIEnvironmentValues: The Git commit sha couldn't be automatically extracted.");
                 }
 
                 // If not set use the default value
@@ -296,11 +328,11 @@ protected override void Setup(IGitInfo gitInfo)
                 {
                     if (string.IsNullOrEmpty(defaultValue))
                     {
-                        Log.Error("DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA must be a full-length git SHA, and the The Git commit sha couldn't be automatically extracted.");
+                        Log.Error("CIEnvironmentValues: DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA must be a full-length git SHA ({Value}), and the The Git commit sha couldn't be automatically extracted.", value);
                     }
                     else
                     {
-                        Log.Error("DD_GIT_CODD_GIT_PULL_REQUEST_BASE_BRANCH_SHAMMIT_SHA must be a full-length git SHA, defaulting to '{Default}", defaultValue);
+                        Log.Error("CIEnvironmentValues: DD_GIT_CODD_GIT_PULL_REQUEST_BASE_BRANCH_SHAMMIT_SHA must be a full-length git SHA ({Value}), defaulting to '{Default}", value, defaultValue);
                     }
 
                     return false;

tracer/src/Datadog.Trace/Ci/CiEnvironment/GitCommandGitInfoProvider.cs

@@ -9,6 +9,7 @@
 using System.IO;
 using System.Linq;
 using Datadog.Trace.Logging;
+using Datadog.Trace.Telemetry.Metrics;
 using Datadog.Trace.Util;
 
 namespace Datadog.Trace.Ci.CiEnvironment;
@@ -23,34 +24,21 @@ private GitCommandGitInfoProvider()
 
     protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true)] out IGitInfo? gitInfo)
     {
+        using var cd = CodeDurationRef.Create();
         var localGitInfo = new GitInfo
         {
-            SourceRoot = gitDirectory.Parent?.FullName
+            SourceRoot = gitDirectory.Name == ".git" ? gitDirectory.Parent?.FullName : gitDirectory.FullName
         };
 
         gitInfo = localGitInfo;
 
         try
         {
-            // Ensure we have permissions to read the git directory
-            var safeDirectory = ProcessHelpers.RunCommand(
-                new ProcessHelpers.Command(
-                    cmd: "git",
-                    arguments: $"config --global --add safe.directory {gitDirectory.FullName}",
-                    workingDirectory: gitDirectory.FullName,
-                    useWhereIsIfFileNotFound: true));
-            if (safeDirectory?.ExitCode != 0)
-            {
-                localGitInfo.Errors.Add($"Error setting safe.directory: {safeDirectory?.Error}");
-            }
-
             // Get the repository URL
-            var repositoryOutput = ProcessHelpers.RunCommand(
-                new ProcessHelpers.Command(
-                    cmd: "git",
-                    arguments: "ls-remote --get-url",
-                    workingDirectory: gitDirectory.FullName,
-                    useWhereIsIfFileNotFound: true));
+            var repositoryOutput = GitCommandHelper.RunGitCommand(
+                localGitInfo.SourceRoot,
+                "ls-remote --get-url",
+                MetricTags.CIVisibilityCommands.GetRepository);
             if (repositoryOutput?.ExitCode == 0)
             {
                 localGitInfo.Repository = repositoryOutput.Output.Trim();
@@ -61,12 +49,10 @@ protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true
             }
 
             // Get the branch name
-            var branchOutput = ProcessHelpers.RunCommand(
-                new ProcessHelpers.Command(
-                    cmd: "git",
-                    arguments: "rev-parse --abbrev-ref HEAD",
-                    workingDirectory: gitDirectory.FullName,
-                    useWhereIsIfFileNotFound: true));
+            var branchOutput = GitCommandHelper.RunGitCommand(
+                localGitInfo.SourceRoot,
+                "rev-parse --abbrev-ref HEAD",
+                MetricTags.CIVisibilityCommands.GetBranch);
             if (branchOutput?.ExitCode == 0 && branchOutput.Output.Trim() is { Length: > 0 } branchName && branchName != "HEAD")
             {
                 localGitInfo.Branch = branchName;
@@ -77,12 +63,10 @@ protected override bool TryGetFrom(DirectoryInfo gitDirectory, [NotNullWhen(true
             }
 
             // Get the remaining data from the log -1
-            var gitLogOutput = ProcessHelpers.RunCommand(
-                new ProcessHelpers.Command(
-                    cmd: "git",
-                    arguments: """log -1 --pretty='%H|,|%at|,|%an|,|%ae|,|%ct|,|%cn|,|%ce|,|%B'""",
-                    workingDirectory: gitDirectory.FullName,
-                    useWhereIsIfFileNotFound: true));
+            var gitLogOutput = GitCommandHelper.RunGitCommand(
+                localGitInfo.SourceRoot,
+                """log -1 --pretty='%H|,|%at|,|%an|,|%ae|,|%ct|,|%cn|,|%ce|,|%B'""",
+                MetricTags.CIVisibilityCommands.Fetch);
             if (gitLogOutput?.ExitCode != 0)
             {
                 localGitInfo.Errors.Add($"Error getting git log: {gitLogOutput?.Error}");

tracer/src/Datadog.Trace/Ci/CiEnvironment/GitInfo.cs

@@ -15,8 +15,10 @@ namespace Datadog.Trace.Ci.CiEnvironment;
 internal sealed class GitInfo : IGitInfo
 {
     private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(GitInfo));
+    private static readonly string? RuntimeFolder = Path.GetDirectoryName(typeof(string).Assembly.Location);
+    private static readonly string DatadogTraceToolsRunnerAssembly = "Datadog.Trace.Tools.Runner.dll";
     private static IGitInfoProvider[] _gitInfoProviders = [
-        ManualParserGitInfoProvider.Instance,
+        // ManualParserGitInfoProvider.Instance,
         GitCommandGitInfoProvider.Instance,
     ];
 
@@ -121,8 +123,30 @@ public static IGitInfo GetFrom(string folder)
     public static IGitInfo GetCurrent()
     {
         List<string>? errors = null;
-        var baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
-        var gitDirectory = GetParentGitFolder(baseDirectory) ?? GetParentGitFolder(Environment.CurrentDirectory);
+        List<string> searchPaths = new();
+
+        var baseDirectory = AppContext.BaseDirectory;
+        if (!baseDirectory.Contains("/dotnet/sdk") && !string.IsNullOrWhiteSpace(RuntimeFolder) && !baseDirectory.Contains(RuntimeFolder))
+        {
+            if (!File.Exists(Path.Combine(baseDirectory, DatadogTraceToolsRunnerAssembly)))
+            {
+                searchPaths.Add(baseDirectory);
+            }
+        }
+
+        searchPaths.Add(Environment.CurrentDirectory);
+
+        DirectoryInfo? gitDirectory = null;
+        foreach (var sp in searchPaths)
+        {
+            Log.Information("GitInfo: Using directory: {Directory}", sp);
+            gitDirectory = GetParentGitFolder(sp);
+            if (gitDirectory is not null)
+            {
+                break;
+            }
+        }
+
         if (gitDirectory != null)
         {
             foreach (var provider in _gitInfoProviders)