Include telemetry_forwarder.exe in the Windows SSI OCI images (#7342)

## Summary of changes

Downloads and verifies the telemetry_forwarder.exe binary and includes
it in the OCI image

## Reason for change

We want to ship the binary side-by-side with the fleet installer. At
some point we'll configure a proper release process for the
`telemetry_forwarder.exe` dll, but in the mean time we're simply pulling
from a public location for simplicity.

## Implementation details

- Download the telemetry forwarder from our Azure blob storage
- Sign the binary as part of our normal GitLab build
- Include the forwarder in our GitLab build output
- Set the `DD_TELEMETRY_FORWARDER_PATH` when installing the tracer

## Test coverage

Less than I'd like, but ultimately this is a non-critical component,
that is tested independently. It would be nice to have some testing that
the telemetry is forwarded correctly, but ultimately I think this should
be added in the system-tests/onboarding tests repo. I included the
telemetry forwarder in our smoke tests so we should at least get
warnings/errors there if there's anything egregious.

## Other details

Stacked on 
- #7341

Author: andrewlock

.azure-pipelines/ultimate-pipeline.yml

@@ -1025,8 +1025,8 @@ stages:
           artifact: dd-dotnet-win-x64
           path: $(monitoringHome)/win-x64
 
-      - script: tracer\build.cmd PackageTracerHome PublishFleetInstaller
-        displayName: Build MSI and Tracer home
+      - script: tracer\build.cmd PackageTracerHome PublishFleetInstaller DownloadWinSsiTelemetryForwarder
+        displayName: Build MSI, Tracer home, FleetInstaller, and TelemetryForwarder
         retryCountOnTaskFailure: 1
 
       - publish: $(artifacts)/windows-tracer-home.zip
@@ -1045,6 +1045,10 @@ stages:
         displayName: Publish fleet installer
         artifact: fleet-installer
 
+      - publish: $(artifacts)/telemetry_forwarder.exe
+        displayName: Publish telemetry-forwarder
+        artifact: fleet-installer-telemetry-forwarder
+
 - stage: build_dd_dotnet_windows
   dependsOn: [merge_commit_id]
   variables:
@@ -6048,6 +6052,11 @@ stages:
             artifact: fleet-installer
             path: $(smokeTestAppDir)/artifacts/installer
 
+        - template: steps/download-artifact.yml
+          parameters:
+            artifact: fleet-installer-telemetry-forwarder
+            path: $(smokeTestAppDir)/artifacts/installer
+
         - powershell: |
             ls $(smokeTestAppDir)/artifacts
             ls $(smokeTestAppDir)/artifacts/installer

.gitlab-ci.yml

@@ -37,7 +37,7 @@ build:
         -e SIGN_WINDOWS=true `
         -e NUGET_CERT_REVOCATION_MODE=offline `
         registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:latest `
-        Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi
+        Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder
     - mkdir artifacts-out
     - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out
     - remove-item -recurse -force build-out\${CI_JOB_ID}

.gitlab/prepare-oci-package.sh

@@ -56,6 +56,9 @@ elif [ "$OS" == "windows" ]; then
   mkdir -p sources/installer
   unzip ../artifacts/windows/fleet-installer.zip -d sources/installer/
 
+  # Copy the telemetry forwarder in too
+  cp ../artifacts/windows/telemetry_forwarder.exe sources/installer/
+
 fi
 
 echo -n $VERSION > sources/version

tracer/build/_build/Build.Gitlab.cs

@@ -2,6 +2,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Security.Cryptography;
 using System.Security.Cryptography.X509Certificates;
 using System.Threading.Tasks;
 using Amazon.SimpleSystemsManagement;
@@ -19,6 +22,32 @@
 
 partial class Build
 {
+    Target DownloadWinSsiTelemetryForwarder => _ => _
+       .Description("Downloads the telemetry forwarder executable used by SSI ")
+       .Unlisted()
+       .Requires(() => IsWin)
+       .Before(SignDlls)
+       .Executes(async () =>
+        {
+            // Download the forwarder from Azure for now.
+            // We will likely change this in the future, but it'll do for now.
+            const string url = "https://apmdotnetci.blob.core.windows.net/apm-datadog-win-ssi-telemetry-forwarder/c83ee9ad2f93c7314779051662e2e00086a213e0/telemetry_forwarder.exe";
+            const string expectedHash = "0B192C1901C670FC9A55464AFDF39774AB7CD0D667ECFB37BC22C27184B49C37D4658383E021F792A2F0C7024E1091F35C3CAD046EC68871FAEEE3C98A40163A";
+
+            var tempFile = await DownloadFile(url);
+            var actualHash = GetSha512Hash(tempFile);
+            if (!string.Equals(expectedHash, actualHash, StringComparison.Ordinal))
+            {
+                throw new Exception($"Downloaded file did not have expected hash. Expected hash {expectedHash}, actual hash {actualHash}");
+            }
+
+            Logger.Information("Hash verified: '{Hash}'", expectedHash);
+
+            // Move to expected location
+            var output = ArtifactsDirectory / "telemetry_forwarder.exe";
+            FileSystemTasks.CopyFile(tempFile, output, FileExistsPolicy.Overwrite);
+        });
+
     Target SignDlls => _ => _
        .Description("Sign the dlls produced by building the Tracer, Profiler, and Monitoring home directory, as well as the dd-dotnet exes")
        .Unlisted()

tracer/build/_build/Build.Utilities.cs

@@ -7,6 +7,7 @@
 using System.Net.Http;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Security.Cryptography;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Amazon.SimpleSystemsManagement.Model;
@@ -612,6 +613,76 @@ string DiffToString(Diff diff)
             var lines = diff.text.TrimEnd(trimChar: '\n').Split(Environment.NewLine);
             return string.Join(Environment.NewLine, lines.Select(l => symbol + l));
         }
+    }
+
+    /// <summary>
+    /// Tries to download a file from the provided url, with a retry, and saves it at a temp path
+    /// </summary>
+    /// <param name="url">The URL to download from</param>
+    /// <returns>The temporary path where the file has been saved</returns>
+    /// <exception cref="Exception"></exception>
+    private static async Task<string> DownloadFile(string url)
+    {
+        using var client = new HttpClient();
+        var attemptsRemaining = 3;
+        var defaultDelay = TimeSpan.FromSeconds(2);
+
+        while (attemptsRemaining > 0)
+        {
+            var retryDelay = defaultDelay;
+            try
+            {
+                Logger.Information("Downloading from {Url}", url);
+                using var response = await client.GetAsync(url);
+                var outputPath = Path.GetTempFileName();
+
+                if (response.IsSuccessStatusCode)
+                {
+                    Logger.Information("Saving file to {Path}", outputPath);
+                    await using var fs = new FileStream(outputPath, FileMode.Create, FileAccess.Write);
+                    await response.Content.CopyToAsync(fs);
+                    return outputPath;
+                }
+
+                Logger.Warning("Failed to download file from {Url}, {StatusCode}: {Body}", url, response.StatusCode, await response.Content.ReadAsStringAsync());
+
+                if (response.StatusCode == HttpStatusCode.TooManyRequests
+                    && response.Headers.TryGetValues("Retry-After", out var values)
+                    && values.FirstOrDefault() is {} retryAfter)
+                    {
+                        if (int.TryParse(retryAfter, out var seconds))
+                        {
+                            retryDelay = TimeSpan.FromSeconds(seconds);
+                        }
+                        else if (DateTimeOffset.TryParse(retryAfter, out var retryDate))
+                        {
+                            var delta = retryDate - DateTimeOffset.UtcNow;
+                            retryDelay = delta > TimeSpan.Zero ? delta : retryDelay;
+                        }
+                    }
+            }
+            catch (Exception ex)
+            {
+                Logger.Warning(ex, "Error downloading file from {Url}", url);
+            }
+
+            attemptsRemaining--;
+            if (attemptsRemaining > 0)
+            {
+                Logger.Debug("Waiting {RetryDelayTotalSeconds} seconds before retry...", retryDelay.TotalSeconds);
+                await Task.Delay(retryDelay);
+            }
+        }
+
+        throw new Exception("Failed to download telemetry forwarder");
+    }
+
+    static string GetSha512Hash(string filePath)
+    {
+        using var sha512 = SHA512.Create();
+        using var stream = File.OpenRead(filePath);
 
+        var hashBytes = sha512.ComputeHash(stream);
+        return Convert.ToHexString(hashBytes);
     }
 }

tracer/build/_build/docker/smoke.windows.fleet-installer.dockerfile

@@ -29,10 +29,12 @@ COPY --from=builder /src/artifacts /install
 
 RUN mkdir /logs; \
     mkdir /monitoring-home; \
+    mkdir /installer; \
     cd /install; \
     Expand-Archive 'c:\install\windows-tracer-home.zip' -DestinationPath 'c:\monitoring-home\';  \
-    c:\install\installer\Datadog.FleetInstaller.exe install-version --home-path c:\monitoring-home; \
-    c:\install\installer\Datadog.FleetInstaller.exe enable-global-instrumentation --home-path c:\monitoring-home; \
+    Copy-Item c:\install\installer\*.* -Destination 'c:\installer\'; \
+    c:\installer\Datadog.FleetInstaller.exe install-version --home-path c:\monitoring-home; \
+    c:\installer\Datadog.FleetInstaller.exe enable-global-instrumentation --home-path c:\monitoring-home; \
     cd /app;
 
 # Set the additional env vars

tracer/src/Datadog.FleetInstaller/PathHelper.cs

@@ -0,0 +1,28 @@
+// <copyright file="PathHelper.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System.IO;
+
+namespace Datadog.FleetInstaller;
+
+internal static class PathHelper
+{
+    private const string ForwarderFileName = "telemetry_forwarder.exe";
+
+    public static string GetTelemetryForwarderPath(string homePath)
+    {
+        if (string.IsNullOrEmpty(homePath)
+            || !Path.IsPathRooted(homePath) // can't use relative paths
+            || Path.GetDirectoryName(homePath) is not { } directory)
+        {
+            // I guess we failed for some reason, so we can't calculate the path to the telemetry forwarder
+            return string.Empty;
+        }
+
+        // Ok, now we have the path
+        // Should we care about long paths?
+        return Path.Combine(Path.GetFullPath(directory), "installer", ForwarderFileName);
+    }
+}

tracer/src/Datadog.FleetInstaller/TracerValues.cs

@@ -16,6 +16,7 @@ public TracerValues(string tracerHomeDirectory)
         TracerHomeDirectory = tracerHomeDirectory;
         NativeLoaderX86Path = Path.Combine(tracerHomeDirectory, "win-x86", "Datadog.Trace.ClrProfiler.Native.dll");
         NativeLoaderX64Path = Path.Combine(tracerHomeDirectory, "win-x64", "Datadog.Trace.ClrProfiler.Native.dll");
+        TelemetryForwarderPath = PathHelper.GetTelemetryForwarderPath(tracerHomeDirectory);
         IisRequiredEnvVariables = new(new Dictionary<string, string>
         {
             { "COR_PROFILER", "{846F5F1C-F9AE-4B07-969E-05C26BC060D8}" },
@@ -28,6 +29,7 @@ public TracerValues(string tracerHomeDirectory)
             { "COR_ENABLE_PROFILING", "1" },
             { "CORECLR_ENABLE_PROFILING", "1" },
             { "DD_INJECTION_ENABLED", "tracer" },
+            { "DD_TELEMETRY_FORWARDER_PATH", TelemetryForwarderPath },
             { Defaults.InstrumentationInstallTypeKey, Defaults.InstrumentationInstallTypeValue },
         });
 
@@ -47,6 +49,7 @@ public TracerValues(string tracerHomeDirectory)
             // { "COR_ENABLE_PROFILING", "1" },
             { "CORECLR_ENABLE_PROFILING", "1" },
             { "DD_TRACING_ENABLED", "tracing" },
+            { "DD_TELEMETRY_FORWARDER_PATH", TelemetryForwarderPath },
             { Defaults.InstrumentationInstallTypeKey, Defaults.InstrumentationInstallTypeValue },
         });
 
@@ -63,6 +66,8 @@ public TracerValues(string tracerHomeDirectory)
 
     public string NativeLoaderX64Path { get; }
 
+    public string TelemetryForwarderPath { get; }
+
     public ReadOnlyDictionary<string, string> IisRequiredEnvVariables { get; }
 
     public ReadOnlyDictionary<string, string> GlobalRequiredEnvVariables { get; }

tracer/test/Datadog.Trace.IntegrationTests/FleetInstaller/PathHelperTests.cs

@@ -0,0 +1,41 @@
+// <copyright file="PathHelperTests.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#if NETFRAMEWORK
+
+using Datadog.FleetInstaller;
+using FluentAssertions;
+using Xunit;
+
+namespace Datadog.Trace.IntegrationTests.FleetInstaller;
+
+public class PathHelperTests
+{
+    [Theory]
+    [InlineData(null)]
+    [InlineData("")]
+    [InlineData("./installer")] // relative
+    [InlineData("somewhere/../installer")] // relative
+    [InlineData(@"c:\")] // root (no parent)
+    public void ForInvalidPaths_ReturnsEmptyString(string homePath)
+    {
+        var actual = PathHelper.GetTelemetryForwarderPath(homePath);
+        actual.Should().BeEmpty();
+    }
+
+    [Theory]
+    [InlineData(@"c:\some\path\to\aplace", @"c:\some\path\to\installer\telemetry_forwarder.exe")]
+    [InlineData(@"c:\some\path\..\to\aplace", @"c:\some\to\installer\telemetry_forwarder.exe")]
+    [InlineData(@"D:\some\path\to\aplace.exe", @"D:\some\path\to\installer\telemetry_forwarder.exe")] // a bit questionable, but meh
+    [InlineData(@"c:\library", @"c:\installer\telemetry_forwarder.exe")]
+    [InlineData(@"f:\other-library", @"f:\installer\telemetry_forwarder.exe")]
+    public void ForValidPaths_ReturnsExpectedPath(string homePath, string expected)
+    {
+        var actual = PathHelper.GetTelemetryForwarderPath(homePath);
+        actual.Should().Be(expected);
+    }
+}
+
+#endif