Merge pull request #1555 from buildkite/SUP-4545-log-group-retention-policies

Implement CloudWatch LogGroup retention policies

Author: petetomasik

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -432,6 +432,32 @@ systemctl daemon-reload
 echo Starting buildkite-agent...
 systemctl enable --now buildkite-agent
 
+echo Configuring CloudWatch agent log retention...
+if [[ -n "${EC2_LOG_RETENTION_DAYS:-}" && "${ENABLE_EC2_LOG_RETENTION_POLICY:-false}" == "true" ]]; then
+  echo "Setting CloudWatch EC2 log retention to ${EC2_LOG_RETENTION_DAYS} days"
+  echo "WARNING: This will delete EC2 logs older than ${EC2_LOG_RETENTION_DAYS} days from existing log groups"
+
+  # Update the CloudWatch agent config with the retention value
+  CONFIG_FILE="/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json"
+  if [[ -f "$CONFIG_FILE" ]]; then
+    # Add retention_in_days to all collect_list items using jq
+    jq --arg retention "$EC2_LOG_RETENTION_DAYS" '
+      .logs.logs_collected.files.collect_list |= map(. + {"retention_in_days": ($retention | tonumber)})
+    ' "$CONFIG_FILE" >/tmp/cloudwatch_config.json && mv /tmp/cloudwatch_config.json "$CONFIG_FILE"
+
+    # Restart CloudWatch agent to pick up the new configuration
+    /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "file:$CONFIG_FILE" || echo "Warning: Failed to restart CloudWatch agent"
+    echo "CloudWatch agent configuration updated and restarted"
+  else
+    echo "Warning: CloudWatch agent config file not found at $CONFIG_FILE"
+  fi
+elif [[ -n "${EC2_LOG_RETENTION_DAYS:-}" ]]; then
+  echo "EC2 log retention set to ${EC2_LOG_RETENTION_DAYS} days but EnableEC2LogRetentionPolicy is false"
+  echo "Skipping EC2 log retention configuration to protect existing logs"
+else
+  echo "EC2 log retention not set, using CloudWatch agent defaults (never expire)"
+fi
+
 echo Signaling success to CloudFormation...
 # This will fail if the stack has already completed, for instance if there is a min size
 # of 1 and this is the 2nd instance. This is ok, so we just ignore the error

packer/linux/conf/cloudwatch-agent/config.json

@@ -53,6 +53,12 @@
 						"log_group_name": "/buildkite/auth",
 						"log_stream_name": "{instance_id}",
 						"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+					},
+					{
+						"file_path": "/var/log/lifecycled.log",
+						"log_group_name": "/buildkite/lifecycled",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
 					}
 				]
 			}

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -309,6 +309,38 @@ If ($lastexitcode -ne 0) { Exit $lastexitcode }
 
 Restart-Service buildkite-agent
 
+Write-Output "Configuring CloudWatch agent log retention..."
+if ($Env:EC2_LOG_RETENTION_DAYS -and $Env:ENABLE_EC2_LOG_RETENTION_POLICY -eq "true") {
+  Write-Output "Setting CloudWatch EC2 log retention to $($Env:EC2_LOG_RETENTION_DAYS) days"
+  Write-Output "WARNING: This will delete EC2 logs older than $($Env:EC2_LOG_RETENTION_DAYS) days from existing log groups"
+
+  $configFile = "C:\ProgramData\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent.json"
+  if (Test-Path $configFile) {
+    # Add retention_in_days to all collect_list items using jq
+    $tempFile = "$env:TEMP\cloudwatch_config.json"
+    & jq --arg retention $Env:EC2_LOG_RETENTION_DAYS '
+      .logs.logs_collected.files.collect_list |= map(. + {"retention_in_days": ($retention | tonumber)}) |
+      .logs.logs_collected.windows_events.collect_list |= map(. + {"retention_in_days": ($retention | tonumber)})
+    ' $configFile > $tempFile
+    Move-Item $tempFile $configFile
+
+    # Restart CloudWatch agent to pick up the new configuration
+    try {
+      & "C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1" -a fetch-config -m ec2 -s -c "file:$configFile"
+      Write-Output "CloudWatch agent configuration updated and restarted"
+    } catch {
+      Write-Output "Warning: Failed to restart CloudWatch agent: $($_.Exception.Message)"
+    }
+  } else {
+    Write-Output "Warning: CloudWatch agent config file not found at $configFile"
+  }
+} elseif ($Env:EC2_LOG_RETENTION_DAYS) {
+  Write-Output "EC2 log retention set to $($Env:EC2_LOG_RETENTION_DAYS) days but EnableEC2LogRetentionPolicy is false"
+  Write-Output "Skipping EC2 log retention configuration to protect existing logs"
+} else {
+  Write-Output "EC2 log retention not set, using CloudWatch agent defaults (never expire)"
+}
+
 # renable debug tracing
 Set-PSDebug -Trace 2
 

templates/aws-stack.yml

@@ -59,6 +59,8 @@ Metadata:
         - BuildkiteWindowsAdministrator
         - BuildkiteAgentScalerServerlessARN
         - BuildkiteAgentScalerVersion
+        - EnableEC2LogRetentionPolicy
+        - EC2LogRetentionDays
         - LogRetentionDays
         - BuildkiteAgentEnableGracefulShutdown
 
@@ -236,10 +238,27 @@ Parameters:
     Type: Number
     Default: 3600
 
+  EnableEC2LogRetentionPolicy:
+    Type: String
+    Default: "false"
+    AllowedValues: ["true", "false"]
+    Description: >
+      Enable CloudWatch log retention policy for EC2 instance logs managed by the CloudWatch agent.
+      When enabled, EC2 logs older than EC2LogRetentionDays will be automatically deleted to reduce storage costs.
+      This only affects logs from Buildkite agents, system logs, and other EC2-generated logs - not Lambda or other AWS service logs.
+      WARNING: For existing stacks, this will delete historical EC2 logs older than the retention period. This action cannot be undone.
+
+  EC2LogRetentionDays:
+    Type: Number
+    Description: The number of days to retain CloudWatch Logs for EC2 instances managed by the CloudWatch agent (Buildkite agents, system logs, etc).
+    Default: 7
+    AllowedValues: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653]
+
   LogRetentionDays:
     Type: Number
-    Description: The number of days to retain the Cloudwatch Logs of the lambda.
-    Default: "1"
+    Description: The number of days to retain CloudWatch Logs for Lambda functions in the stack.
+    Default: 1
+    AllowedValues: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653]
 
   BuildkiteAgentEnableGracefulShutdown:
     Description: >
@@ -1358,6 +1377,7 @@ Resources:
               - logs:PutLogEvents
               - logs:DescribeLogGroups
               - logs:DescribeLogStreams
+              - logs:PutRetentionPolicy
             Resource: "*"
           - Sid: Ssm
             Effect: Allow
@@ -1622,6 +1642,8 @@ Resources:
                   $Env:ECR_PLUGIN_ENABLED="${EnableECRPlugin}"
                   $Env:DOCKER_LOGIN_PLUGIN_ENABLED="${EnableDockerLoginPlugin}"
                   $Env:AWS_REGION="${AWS::Region}"
+                  $Env:ENABLE_EC2_LOG_RETENTION_POLICY="${EnableEC2LogRetentionPolicy}"
+                  $Env:EC2_LOG_RETENTION_DAYS="${EC2LogRetentionDays}"
                   powershell -file C:\buildkite-agent\bin\bk-install-elastic-stack.ps1 >> C:\buildkite-agent\elastic-stack.log
                   </powershell>
                 - LocalSecretsBucket: !If
@@ -1721,6 +1743,8 @@ Resources:
                   RESOURCE_LIMITS_CPU_WEIGHT="${ResourceLimitsCPUWeight}" \
                   RESOURCE_LIMITS_CPU_QUOTA="${ResourceLimitsCPUQuota}" \
                   RESOURCE_LIMITS_IO_WEIGHT="${ResourceLimitsIOWeight}" \
+                  ENABLE_EC2_LOG_RETENTION_POLICY="${EnableEC2LogRetentionPolicy}" \
+                  EC2_LOG_RETENTION_DAYS="${EC2LogRetentionDays}" \
                     /usr/local/bin/bk-install-elastic-stack.sh
                   --==BOUNDARY==--
                 - LocalSecretsBucket: !If