Merge pull request #1030 from buildkite/tracing-backends

Make tracing easier to configure

Author: moskyb

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -182,8 +182,13 @@ spawn=${BUILDKITE_AGENTS_PER_INSTANCE}
 no-color=true
 disconnect-after-idle-timeout=${BUILDKITE_SCALE_IN_IDLE_PERIOD}
 disconnect-after-job=${BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB}
+tracing-backend=${BUILDKITE_AGENT_TRACING_BACKEND}
 EOF
 
+if [[ "${BUILDKITE_ENV_FILE_URL}" != "" ]]; then
+	/usr/local/bin/bk-fetch.sh "${BUILDKITE_ENV_FILE_URL}" /var/lib/buildkite-agent/env
+fi
+
 chown buildkite-agent: /etc/buildkite-agent/buildkite-agent.cfg
 
 if [[ -n "${BUILDKITE_AUTHORIZED_USERS_URL}" ]] ; then

packer/linux/conf/buildkite-agent/systemd/buildkite-agent.service

@@ -11,6 +11,8 @@ User=buildkite-agent
 Environment="HOME=/var/lib/buildkite-agent"
 Environment="PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"
 Environment="USER=buildkite-agent"
+# The =- (rather than just = ) in the line below means that systemd won't complain if it can't find the file
+EnvironmentFile=-/var/lib/buildkite-agent/env
 ExecStart=/usr/bin/buildkite-agent start
 ExecStopPost=/usr/local/bin/terminate-instance
 RestartSec=5

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -138,6 +138,7 @@ no-color=true
 shell=powershell
 disconnect-after-idle-timeout=${Env:BUILDKITE_SCALE_IN_IDLE_PERIOD}
 disconnect-after-job=${Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB}
+tracing-backend=${Env:BUILDKITE_AGENT_TRACING_BACKEND}
 "@
 $OFS=" "
 
@@ -199,6 +200,10 @@ If (![string]::IsNullOrEmpty($Env:BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT)) {
   Remove-Item -Path C:\buildkite-agent\elastic_bootstrap.ps1
 }
 
+If (![string]::IsNullOrEmpty($Env:BUILDKITE_ENV_FILE_URL)) {
+  C:\buildkite-agent\bin\bk-fetch.ps1 -From "$Env:BUILDKITE_ENV_FILE_URL" -To C:\buildkite-agent\env
+}
+
 Write-Output "Starting the Buildkite Agent"
 
 nssm install buildkite-agent C:\buildkite-agent\bin\buildkite-agent.exe start
@@ -210,6 +215,14 @@ If ($lastexitcode -ne 0) { Exit $lastexitcode }
 nssm set buildkite-agent AppStderr C:\buildkite-agent\buildkite-agent.log
 If ($lastexitcode -ne 0) { Exit $lastexitcode }
 nssm set buildkite-agent AppEnvironmentExtra :HOME=C:\buildkite-agent
+
+If ((![string]::IsNullOrEmpty($Env:BUILDKITE_ENV_FILE_URL)) -And (Test-Path -Path C:\buildkite-agent\env -PathType leaf)) {
+  foreach ($var in Get-Content C:\buildkite-agent\env) {
+    nssm set buildkite-agent AppEnvironmentExtra $var
+    If ($lastexitcode -ne 0) { Exit $lastexitcode }
+  }
+}
+
 If ($lastexitcode -ne 0) { Exit $lastexitcode }
 nssm set buildkite-agent AppExit Default Restart
 If ($lastexitcode -ne 0) { Exit $lastexitcode }

templates/aws-stack.yml

@@ -41,6 +41,7 @@ Metadata:
         - BuildkiteAgentTags
         - BuildkiteAgentTimestampLines
         - BuildkiteAgentExperiments
+        - BuildkiteAgentTracingBackend
         - BuildkiteTerminateInstanceAfterJob
         - BuildkiteAdditionalSudoPermissions
         - BuildkiteWindowsAdministrator
@@ -69,6 +70,7 @@ Metadata:
         - ArtifactsBucket
         - AuthorizedUsersUrl
         - BootstrapScriptUrl
+        - AgentEnvFileUrl
         - RootVolumeSize
         - RootVolumeName
         - RootVolumeType
@@ -163,6 +165,15 @@ Parameters:
     Type: String
     Default: ""
 
+  BuildkiteAgentTracingBackend:
+    Description: The tracing backend to use for CI tracing. See https://buildkite.com/docs/agent/v3/tracing
+    Type: String
+    AllowedValues:
+      - ""
+      - "datadog"
+      - "opentelemetry"
+    Default: ""
+
   BuildkiteTerminateInstanceAfterJob:
     Description: Set to "true" to terminate the instance after a job has completed.
     Type: String
@@ -212,7 +223,12 @@ Parameters:
     Default: ""
 
   BootstrapScriptUrl:
-    Description: Optional - HTTPS or S3 URL to run on each instance during boot
+    Description: Optional - HTTPS or S3 URL for a script to run on each instance during boot
+    Type: String
+    Default: ""
+
+  AgentEnvFileUrl:
+    Description: Optional - HTTPS or S3 URL for a list of environment variables to propagate into the agent's execution environment. Note that these environment variables **will not** be passed into builds running on the agent, only to the agent process itself.
     Type: String
     Default: ""
 
@@ -443,7 +459,7 @@ Parameters:
       - "true"
       - "false"
     Default: "false"
-    
+
   EnableDetailedMonitoring:
     Type: String
     Description: Enable detailed EC2 monitoring
@@ -993,10 +1009,12 @@ Resources:
                   $Env:BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}"
                   $Env:BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}"
                   $Env:BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}"
+                  $Env:BUILDKITE_AGENT_TRACING_BACKEND="${BuildkiteAgentTracingBackend}"
                   $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}"
                   $Env:BUILDKITE_QUEUE="${BuildkiteQueue}"
                   $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}"
                   $Env:BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}"
+                  $Env:BUILDKITE_ENV_FILE_URL="${AgentEnvFileUrl}"
                   $Env:BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}"
                   $Env:BUILDKITE_ECR_POLICY="${ECRAccessPolicy}"
                   $Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}"
@@ -1041,10 +1059,12 @@ Resources:
                   BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}" \
                   BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" \
                   BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" \
+                  BUILDKITE_AGENT_TRACING_BACKEND="${BuildkiteAgentTracingBackend}" \
                   BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" \
                   BUILDKITE_QUEUE="${BuildkiteQueue}" \
                   BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}" \
                   BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" \
+                  BUILDKITE_ENV_FILE_URL=${AgentEnvFileUrl} \
                   BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \
                   BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" \
                   BUILDKITE_ECR_POLICY="${ECRAccessPolicy}" \