diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f524f96ace9a..89b54433f0ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: DOTNET_PACKAGE_VERSION: description: "Used by the package stage when triggered manually" REPO_NOTIFICATION_CHANNEL: "#apm-dotnet-bots" + CI_IDENTITIES_CLIENT_URL: s3://binaries-ddbuild-io-prod/ci-identities/ci-identities-gitlab-job-client/versions/v0.2.0/ci-identities-gitlab-job-client-windows-amd64.exe build: except: @@ -37,7 +38,9 @@ build: -e AWS_NETWORKING=true ` -e SIGN_WINDOWS=true ` -e NUGET_CERT_REVOCATION_MODE=offline ` - registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:dotnet10 ` + -e CI_IDENTITIES_GITLAB_ID_TOKEN ` + -e CI_PROJECT_NAME ` + registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:ci-identities ` Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder - mkdir artifacts-out - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out @@ -48,6 +51,9 @@ build: expire_in: 2 weeks paths: - artifacts-out + id_tokens: + CI_IDENTITIES_GITLAB_ID_TOKEN: + aud: ci-identities publish: only: @@ -67,19 +73,17 @@ publish: pre_get_sources_script: - git config --system core.longpaths true script: - - $result = aws sts assume-role --role-arn "arn:aws:iam::486234852809:role/ci-datadog-windows-filter" --role-session-name AWSCLI-Session - - $resultjson = $result | convertfrom-json - - $credentials = $($resultjson.Credentials) - - $Env:AWS_ACCESS_KEY_ID="$($credentials.AccessKeyId)" - - $Env:AWS_SECRET_ACCESS_KEY="$($credentials.SecretAccessKey)" - - $Env:AWS_SESSION_TOKEN="$($credentials.SessionToken)" + # TODO remove the aws s3 cp command + # when the client is installed in the Windows runner image + - aws s3 cp --only-show-errors ${CI_IDENTITIES_CLIENT_URL} ./ci-identities-gitlab-job-client.exe + - ./ci-identities-gitlab-job-client.exe assume-role - | $i = 0 do { try { # The grants option at the end is used to allow public access on the files we upload as the acls only aren't enough. aws s3 cp artifacts-out/ s3://dd-windowsfilter/builds/tracer/${CI_COMMIT_SHA} --recursive --region us-east-1 --exclude "*" --include "*.zip" --include "*.msi" --include "telemetry_forwarder.exe" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 - If ($LASTEXITCODE -eq 0) { + If ($LASTEXITCODE -eq 0) { return } @@ -95,6 +99,11 @@ publish: # If we got here, all retries failed – fail the job: Write-Error "Failed to upload artifacts to S3 after $i attempts." exit 1 + variables: + AWS_SHARED_CREDENTIALS_FILE: ${CI_PROJECT_DIR}\.aws\credentials-by-job-id\${CI_JOB_ID} + id_tokens: + CI_IDENTITIES_GITLAB_ID_TOKEN: + aud: ci-identities download-single-step-artifacts: @@ -245,4 +254,4 @@ validate_supported_configurations_local_file: - when: on_success extends: .validate_supported_configurations_local_file variables: - LOCAL_JSON_PATH: "tracer/src/Datadog.Trace/Configuration/supported-configurations.json" \ No newline at end of file + LOCAL_JSON_PATH: "tracer/src/Datadog.Trace/Configuration/supported-configurations.json" diff --git a/tracer/build/_build/Build.Gitlab.cs b/tracer/build/_build/Build.Gitlab.cs index b411a7fd48ae..3bd1ae4b2dc3 100644 --- a/tracer/build/_build/Build.Gitlab.cs +++ b/tracer/build/_build/Build.Gitlab.cs @@ -88,7 +88,15 @@ partial class Build void SignFiles(IReadOnlyCollection filesToSign) { - const string validSignature = "59063C826DAA5B628B5CE8A2B32015019F164BF0"; + // See list of certificates + // in https://datadoghq.atlassian.net/wiki/spaces/SECENG/pages/3217261499/Certificates+for+Windows+Code+Signing + var expectedCertificateThumbprints = new [] + { + "A0FB7BEE153FE31431062731306903B3A5CB1824", + // TODO remove this one when the new certificate is deployed; + // see https://github.com/DataDog/windows-code-signing-cert/blob/main/current-certs.toml + "59063C826DAA5B628B5CE8A2B32015019F164BF0", + }; Logger.Information("Signing {Count} binaries...", filesToSign.Count); filesToSign.ForEach(file => SignBinary(file)); @@ -99,7 +107,7 @@ void SignBinary(AbsolutePath binaryPath) Logger.Information("Signing {BinaryPath}", binaryPath); var signProcess = ProcessTasks.StartProcess( - "dd-wcs", + "c:/devtools/windows-code-signer.exe", $"sign {binaryPath}", logOutput: false, logInvocation: false); @@ -108,13 +116,7 @@ void SignBinary(AbsolutePath binaryPath) var output = signProcess.Output.Select(o => o.Text); foreach (var line in output) { - Logger.Information("[dd-wcs] {Line}", line); - - // dd-wcs will return 0 even if there are errors - if (line.StartsWith("ERROR:", StringComparison.OrdinalIgnoreCase)) - { - throw new Exception($"Error found when signing {binaryPath}: {line}"); - } + Logger.Information("[windows-code-signer] {Line}", line); } if (signProcess.ExitCode == 0) @@ -138,7 +140,7 @@ void SignBinary(AbsolutePath binaryPath) var printValue = print.Select(o => o.Text).FirstOrDefault(l => !string.IsNullOrEmpty(l))?.Trim(); - if (!string.Equals(printValue, validSignature, StringComparison.OrdinalIgnoreCase)) + if (!expectedCertificateThumbprints.Contains(printValue, StringComparer.OrdinalIgnoreCase)) { throw new Exception($"Signature verification failed for {binaryPath}. Signature: {printValue ?? "Empty"}"); } diff --git a/tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md b/tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md index 5c176b0ee790..6f5a435e1303 100644 --- a/tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md +++ b/tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md @@ -48,6 +48,13 @@ docker inspect --format='{{index .RepoDigests 0}}' datadog/dd-trace-dotnet-docke Extract the SHA256 hash from the output (format: `docker.io/datadog/dd-trace-dotnet-docker-build@sha256:`). +You can also use [crane](https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane.md) + +``` +$ crane digest datadog/dd-trace-dotnet-docker-build:dotnet10-rc1 +sha256:180cb096b25d9c53e24b23d0324cd403cc7fe4e99c88ec2c20e851dc37d359ef +``` + ### 6. Create Mirror PR In the `DataDog/images` repository, add entries to two files: diff --git a/tracer/build/_build/docker/gitlab/entrypoint.bat b/tracer/build/_build/docker/gitlab/entrypoint.bat index a5717cc804ab..d40682245bed 100644 --- a/tracer/build/_build/docker/gitlab/entrypoint.bat +++ b/tracer/build/_build/docker/gitlab/entrypoint.bat @@ -36,6 +36,10 @@ if "%nuke_args%"=="" ( exit /b 1 ) +:: the CI Identities client will write the credentials to the path in the environment variable AWS_SHARED_CREDENTIALS_FILE, +:: and if the variable is not set, it will write to %USERPROFILE%\.aws\credentials +c:\devtools\ci-identities-gitlab-job-client.exe assume-role + dotnet run --project tracer/build/_build/_build.csproj -- %nuke_args% --Artifacts "build-out\%CI_JOB_ID%" IF %ERRORLEVEL% NEQ 0 EXIT /B %ERRORLEVEL% diff --git a/tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile b/tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile index f2650b8596cc..026ab39af930 100644 --- a/tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile +++ b/tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile @@ -40,27 +40,19 @@ ENV DOTNET_VERSION="10.0.100" \ COPY install_dotnet.ps1 . RUN powershell -Command .\install_dotnet.ps1 -Version $ENV:DOTNET_VERSION -Sha512 $ENV:DOTNET_SHA512 $ENV:DOTNET_DOWNLOAD_URL -# Java and code signing tool environment variables -ENV JAVA_VERSION "17.0.8" -ENV JAVA_SHA256 "db6e7e7506296b8a2338f6047fdc94bf4bbc147b7a3574d9a035c3271ae1a92b" -ENV WINSIGN_VERSION "0.3.5" -ENV WINSIGN_SHA256 "b2ba5127a5c5141e04d42444ca115af4c95cc053a743caaa9b33c68dd6b13f68" -ENV PYTHON_VERSION "3.8.2" - -# Install Python -COPY install_python3.ps1 . -RUN powershell -Command .\install_python3.ps1 -Version $ENV:PYTHON_VERSION +# Copy the CI Identities GitLab Job Client +COPY --from=registry.ddbuild.io/ci-identities/ci-identities-gitlab-job-client:v0.2.0-windows-amd64 C:/ci-identities-gitlab-job-client.exe c:/devtools/ci-identities-gitlab-job-client.exe -COPY requirements.txt constraints.txt install_python_packages.ps1 ./ -RUN powershell -Command .\install_python_packages.ps1 +# Java and code signing tool environment variables +ENV JAVA_VERSION "25.0.1" +ENV JAVA_SHA256 "d56bed274adb2b16deea2dce3f21718d1b0dcdbe2253bc5cc332b525cbcd1fd1" # Install JAVA COPY helpers.ps1 install_java.ps1 ./ RUN powershell -Command .\install_java.ps1 -# Install -COPY install_winsign.ps1 . -RUN powershell -Command .\install_winsign.ps1 +# Install Windows Code Signer +COPY --from=registry.ddbuild.io/windows-code-signer/go:v0.6.0-ltsc2019 c:/windows-code-signer/windows-code-signer.exe c:/devtools/windows-code-signer.exe # Copy everything else COPY . . diff --git a/tracer/build/_build/docker/gitlab/install_java.ps1 b/tracer/build/_build/docker/gitlab/install_java.ps1 index 6253f54d3da2..1976630a5b50 100644 --- a/tracer/build/_build/docker/gitlab/install_java.ps1 +++ b/tracer/build/_build/docker/gitlab/install_java.ps1 @@ -44,9 +44,9 @@ Write-Host -ForegroundColor Green 'java --version'; java --version ## need to have more rigorous download at some point, but #$jsignjarsrc = "https://s3.amazonaws.com/dd-agent-omnibus/jsign/jsign-4.2.jar" -$jsignjarsrc = "https://github.com/ebourg/jsign/releases/download/5.0/jsign-5.0.jar" +$jsignjarsrc = "https://github.com/ebourg/jsign/releases/download/7.4/jsign-7.4.jar" $jsignjardir = "c:\devtools\jsign" -$jsignout = "$($jsignjardir)\jsign-5.0.jar" +$jsignout = "$($jsignjardir)\jsign-7.4.jar" if (-Not (test-path $jsignjardir)) { mkdir $jsignjardir } diff --git a/tracer/build/_build/docker/gitlab/install_python3.ps1 b/tracer/build/_build/docker/gitlab/install_python3.ps1 deleted file mode 100644 index f9efe9a8855e..000000000000 --- a/tracer/build/_build/docker/gitlab/install_python3.ps1 +++ /dev/null @@ -1,35 +0,0 @@ -param ( - [Parameter(Mandatory=$true)][string]$Version -) - -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -$ApplicationName = "Python" -$Installer = "$($PSScriptRoot)\python.exe" -$DownloadUrl = "https://www.python.org/ftp/python/$Version/python-$Version-amd64.exe" -$InstallDir = "c:\tmp\Python38" -$PythonScriptsDir = "c:\tmp\Python38\scripts" - -Write-Host -ForegroundColor Green "========= Installing Python $Version =========" - -# Download installer -Write-Host "Downloading $ApplicationName" -(New-Object System.Net.WebClient).DownloadFile($DownloadUrl, $Installer) - -# Start the installer -Write-Host "Installing $ApplicationName" -Start-Process $Installer -ArgumentList '/quiet InstallAllUsers=1 TargetDir=C:\tmp\Python38' -Wait - -# Test to make sure the application actually installed. -if (!(Test-Path $InstallDir)) { - throw "FATAL: '$ApplicationName' was not found after MSI installation" -} -else { - # Cleanup - Remove-Item $Installer - # Add application to PATH - [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$InstallDir;$PythonScriptsDir", [EnvironmentVariableTarget]::Machine) - Write-Host -ForegroundColor Green "$ApplicationName has been installed successfully" -} \ No newline at end of file diff --git a/tracer/build/_build/docker/gitlab/install_python_packages.ps1 b/tracer/build/_build/docker/gitlab/install_python_packages.ps1 deleted file mode 100644 index d963d8133325..000000000000 --- a/tracer/build/_build/docker/gitlab/install_python_packages.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -Write-Host -ForegroundColor Green "========= Installing python packages =========" -& python -m pip install -r "$($PSScriptRoot)\requirements.txt" \ No newline at end of file diff --git a/tracer/build/_build/docker/gitlab/install_winsign.ps1 b/tracer/build/_build/docker/gitlab/install_winsign.ps1 deleted file mode 100644 index d41ff197f446..000000000000 --- a/tracer/build/_build/docker/gitlab/install_winsign.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -$TargetContainer = $true -. "$($PSScriptRoot)\helpers.ps1" -Write-Host -ForegroundColor Green "Installing Windows Codesign Helper $ENV:WINSIGN_VERSION" - -## need to have more rigorous download at some point, but -$codesign_base = "windows_code_signer-$($ENV:WINSIGN_VERSION)-py3-none-any.whl" -$codesign_wheel = "https://s3.amazonaws.com/dd-agent-omnibus/windows-code-signer/$($codesign_base)" -$codesign_wheel_target = "c:\devtools\$($codesign_base)" -(New-Object System.Net.WebClient).DownloadFile($codesign_wheel, $codesign_wheel_target) - -Get-RemoteFile -RemoteFile $codesign_wheel -LocalFile $codesign_wheel_target -VerifyHash $ENV:WINSIGN_SHA256 - -python -m pip install $codesign_wheel_target \ No newline at end of file