[SINT-4550] Use CI Identities in Windows CI Jobs

.gitlab-ci.yml

@@ -13,6 +13,7 @@ variables:
   DOTNET_PACKAGE_VERSION:
     description: "Used by the package stage when triggered manually"
   REPO_NOTIFICATION_CHANNEL: "#apm-dotnet-bots"
+  CI_IDENTITIES_CLIENT_URL: s3://binaries-ddbuild-io-prod/ci-identities/ci-identities-gitlab-job-client/versions/v0.2.0/ci-identities-gitlab-job-client-windows-amd64.exe
 
 build:
   except:
@@ -37,7 +38,9 @@ build:
         -e AWS_NETWORKING=true `
         -e SIGN_WINDOWS=true `
         -e NUGET_CERT_REVOCATION_MODE=offline `
-        registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:dotnet10 `
+        -e CI_IDENTITIES_GITLAB_ID_TOKEN `
+        -e CI_PROJECT_NAME `
+        registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:ci-identities `
         Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder
     - mkdir artifacts-out
     - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out
@@ -48,6 +51,9 @@ build:
     expire_in: 2 weeks
     paths:
     - artifacts-out
+  id_tokens:
+    CI_IDENTITIES_GITLAB_ID_TOKEN:
+      aud: ci-identities
 
 publish:
   only:
@@ -67,19 +73,17 @@ publish:
     pre_get_sources_script:
       - git config --system core.longpaths true
   script:
-    - $result =  aws sts assume-role --role-arn "arn:aws:iam::486234852809:role/ci-datadog-windows-filter" --role-session-name AWSCLI-Session
-    - $resultjson = $result | convertfrom-json
-    - $credentials = $($resultjson.Credentials)
-    - $Env:AWS_ACCESS_KEY_ID="$($credentials.AccessKeyId)"
-    - $Env:AWS_SECRET_ACCESS_KEY="$($credentials.SecretAccessKey)"
-    - $Env:AWS_SESSION_TOKEN="$($credentials.SessionToken)"
+    # TODO remove the aws s3 cp command
+    # when the client is installed in the Windows runner image
+    - aws s3 cp --only-show-errors ${CI_IDENTITIES_CLIENT_URL} ./ci-identities-gitlab-job-client.exe
+    - ./ci-identities-gitlab-job-client.exe assume-role
     - |
       $i = 0
       do {
           try {
               # The grants option at the end is used to allow public access on the files we upload as the acls only aren't enough.
               aws s3 cp artifacts-out/ s3://dd-windowsfilter/builds/tracer/${CI_COMMIT_SHA} --recursive --region us-east-1 --exclude "*" --include "*.zip" --include "*.msi" --include "telemetry_forwarder.exe" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
-              If ($LASTEXITCODE -eq 0) { 
+              If ($LASTEXITCODE -eq 0) {
                 return
               }
 
@@ -95,6 +99,11 @@ publish:
       # If we got here, all retries failed – fail the job:
       Write-Error "Failed to upload artifacts to S3 after $i attempts."
       exit 1
+  variables:
+    AWS_SHARED_CREDENTIALS_FILE: ${CI_PROJECT_DIR}\.aws\credentials-by-job-id\${CI_JOB_ID}
+  id_tokens:
+    CI_IDENTITIES_GITLAB_ID_TOKEN:
+      aud: ci-identities
 
 
 download-single-step-artifacts:
@@ -245,4 +254,4 @@ validate_supported_configurations_local_file:
     - when: on_success
   extends: .validate_supported_configurations_local_file
   variables:
-    LOCAL_JSON_PATH: "tracer/src/Datadog.Trace/Configuration/supported-configurations.json"
+    LOCAL_JSON_PATH: "tracer/src/Datadog.Trace/Configuration/supported-configurations.json"

tracer/build/_build/Build.Gitlab.cs

@@ -88,7 +88,15 @@ partial class Build
 
     void SignFiles(IReadOnlyCollection<AbsolutePath> filesToSign)
     {
-        const string validSignature = "59063C826DAA5B628B5CE8A2B32015019F164BF0";
+        // See list of certificates
+        // in https://datadoghq.atlassian.net/wiki/spaces/SECENG/pages/3217261499/Certificates+for+Windows+Code+Signing
+        var expectedCertificateThumbprints = new []
+        {
+            "A0FB7BEE153FE31431062731306903B3A5CB1824",
+            // TODO remove this one when the new certificate is deployed;
+            // see https://github.com/DataDog/windows-code-signing-cert/blob/main/current-certs.toml
+            "59063C826DAA5B628B5CE8A2B32015019F164BF0",
+        };
 
         Logger.Information("Signing {Count} binaries...", filesToSign.Count);
         filesToSign.ForEach(file => SignBinary(file));
@@ -99,7 +107,7 @@ void SignBinary(AbsolutePath binaryPath)
             Logger.Information("Signing {BinaryPath}", binaryPath);
 
             var signProcess = ProcessTasks.StartProcess(
-                    "dd-wcs",
+                    "c:/devtools/windows-code-signer.exe",
                     $"sign {binaryPath}",
                     logOutput: false,
                     logInvocation: false);
@@ -108,13 +116,7 @@ void SignBinary(AbsolutePath binaryPath)
             var output = signProcess.Output.Select(o => o.Text);
             foreach (var line in output)
             {
-                Logger.Information("[dd-wcs] {Line}", line);
-
-                // dd-wcs will return 0 even if there are errors
-                if (line.StartsWith("ERROR:", StringComparison.OrdinalIgnoreCase))
-                {
-                    throw new Exception($"Error found when signing {binaryPath}: {line}");
-                }
+                Logger.Information("[windows-code-signer] {Line}", line);
             }
 
             if (signProcess.ExitCode == 0)
@@ -138,7 +140,7 @@ void SignBinary(AbsolutePath binaryPath)
 
                 var printValue = print.Select(o => o.Text).FirstOrDefault(l => !string.IsNullOrEmpty(l))?.Trim();
 
-                if (!string.Equals(printValue, validSignature, StringComparison.OrdinalIgnoreCase))
+                if (!expectedCertificateThumbprints.Contains(printValue, StringComparer.OrdinalIgnoreCase))
                 {
                     throw new Exception($"Signature verification failed for {binaryPath}. Signature: {printValue ?? "Empty"}");
                 }

tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md

@@ -48,6 +48,13 @@ docker inspect --format='{{index .RepoDigests 0}}' datadog/dd-trace-dotnet-docke
 
 Extract the SHA256 hash from the output (format: `docker.io/datadog/dd-trace-dotnet-docker-build@sha256:<HASH>`).
 
+You can also use [crane](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane.md)
+
+```
+$ crane digest datadog/dd-trace-dotnet-docker-build:dotnet10-rc1
+sha256:180cb096b25d9c53e24b23d0324cd403cc7fe4e99c88ec2c20e851dc37d359ef
+```
+
 ### 6. Create Mirror PR
 
 In the `DataDog/images` repository, add entries to two files:

tracer/build/_build/docker/gitlab/entrypoint.bat

@@ -36,6 +36,10 @@ if "%nuke_args%"=="" (
     exit /b 1
 )
 
+:: the CI Identities client will write the credentials to the path in the environment variable AWS_SHARED_CREDENTIALS_FILE,
+:: and if the variable is not set, it will write to %USERPROFILE%\.aws\credentials
+c:\devtools\ci-identities-gitlab-job-client.exe assume-role
+
 dotnet run --project tracer/build/_build/_build.csproj -- %nuke_args% --Artifacts "build-out\%CI_JOB_ID%"
 
 IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL%

tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile

@@ -40,27 +40,19 @@ ENV DOTNET_VERSION="10.0.100" \
 COPY install_dotnet.ps1 .
 RUN powershell -Command .\install_dotnet.ps1  -Version $ENV:DOTNET_VERSION -Sha512 $ENV:DOTNET_SHA512 $ENV:DOTNET_DOWNLOAD_URL
 
-# Java and code signing tool environment variables
-ENV JAVA_VERSION "17.0.8"
-ENV JAVA_SHA256 "db6e7e7506296b8a2338f6047fdc94bf4bbc147b7a3574d9a035c3271ae1a92b"
-ENV WINSIGN_VERSION "0.3.5"
-ENV WINSIGN_SHA256 "b2ba5127a5c5141e04d42444ca115af4c95cc053a743caaa9b33c68dd6b13f68"
-ENV PYTHON_VERSION "3.8.2"
-
-# Install Python
-COPY install_python3.ps1 .
-RUN powershell -Command .\install_python3.ps1 -Version $ENV:PYTHON_VERSION
+# Copy the CI Identities GitLab Job Client
+COPY --from=registry.ddbuild.io/ci-identities/ci-identities-gitlab-job-client:v0.2.0-windows-amd64 C:/ci-identities-gitlab-job-client.exe c:/devtools/ci-identities-gitlab-job-client.exe
 
-COPY requirements.txt constraints.txt install_python_packages.ps1 ./
-RUN powershell -Command .\install_python_packages.ps1
+# Java and code signing tool environment variables
+ENV JAVA_VERSION "25.0.1"
+ENV JAVA_SHA256 "d56bed274adb2b16deea2dce3f21718d1b0dcdbe2253bc5cc332b525cbcd1fd1"
 
 # Install JAVA
 COPY helpers.ps1 install_java.ps1 ./
 RUN powershell -Command .\install_java.ps1
 
-# Install 
-COPY install_winsign.ps1 .
-RUN powershell -Command .\install_winsign.ps1
+# Install Windows Code Signer
+COPY --from=registry.ddbuild.io/windows-code-signer/go:v0.6.0-ltsc2019 c:/windows-code-signer/windows-code-signer.exe c:/devtools/windows-code-signer.exe
 
 # Copy everything else
 COPY . .

tracer/build/_build/docker/gitlab/install_java.ps1

@@ -44,9 +44,9 @@ Write-Host -ForegroundColor Green 'java --version'; java --version
 
 ## need to have more rigorous download at some point, but
 #$jsignjarsrc = "https://s3.amazonaws.com/dd-agent-omnibus/jsign/jsign-4.2.jar"
-$jsignjarsrc = "https://github.com/ebourg/jsign/releases/download/5.0/jsign-5.0.jar"
+$jsignjarsrc = "https://github.com/ebourg/jsign/releases/download/7.4/jsign-7.4.jar"
 $jsignjardir = "c:\devtools\jsign"
-$jsignout = "$($jsignjardir)\jsign-5.0.jar"
+$jsignout = "$($jsignjardir)\jsign-7.4.jar"
 if (-Not (test-path $jsignjardir)) {
     mkdir $jsignjardir
 }