add the log-collector to the utils folder and update README

Author: jeremybumsted

README.md

@@ -175,13 +175,17 @@ Feel free to drop an email to [email protected] with questions. It helps us
 aws cloudformation describe-stacks --stack-name MY_STACK_NAME \
   --query 'Stacks[].Parameters[].[ParameterKey,ParameterValue]' --output table
 ```
-
+### Collect logs from CloudWatch
 Provide us with logs from CloudWatch Logs:
 
 ```
 /buildkite/elastic-stack/{instance-id}
 /buildkite/systemd/{instance-id}
 ```
+### Collect logs via script
+An alternative method to collect the logs is to use the `log-collector` script in the `utils` folder.
+The script will collect CloudWatch logs for the Instance, Lambda function, and AutoScaling activity and package them in a
+zip archive which you can send via email to [email protected].
 
 You can also drop by `#aws-stack` and `#aws` channels in [Buildkite Community Slack](https://chat.buildkite.com/) and ask your question!
 

utils/log-collector

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# This script leverages the AWS CLI to collect Elastic Stack logs for your
+# Buildkite Elastic CI Stack instances.
+#
+# It assumes you have the AWS CLI tool configured, with appropriate credentials
+# so it is recommended to use something like aws-vault: https://github.com/99designs/aws-vault.
+#
+# You will need to know the Instance ID and Stack name of the resources
+# you want to collect logs for.
+# 
+# Once you have collected the logs, please send them to [email protected]
+# with a description of the issue you are seeing, and we'll be happy to assist!
+
+set -euo pipefail
+
+usage() {
+cat << EOF
+Usage: ./bk-log-collector -s <stack-name> -i <instance-id>
+Collect Buildkite Elastic CI stack logs from AWS CloudWatch
+-s      Auto Scaling Group Name, can be found at console.aws.amazon.com/ec2autoscaling/
+-i      Instance ID, the AWS instance ID for the instance you want to collect logs for
+-o      Output format for logs(optional flag), supported values are text (default value), json, table
+EOF
+}
+
+while getopts 'hs:i:o:' opt; do
+    case ${opt} in
+        s ) 
+            stackName=$OPTARG
+            ;;
+        i )
+            instanceId=$OPTARG
+            ;;
+        o )
+            outputFormat=$OPTARG
+            ;;
+        h )
+            usage
+            exit 0
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an arguemnt" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+now=$(date +"%F")
+outputFormat="${outputFormat:=text}"
+
+# Create a temporary directory to store our logs
+pwd=$(pwd)
+dir=$(mktemp -d "$pwd"/buildkite-logs-"$now")
+
+echo "Collecting logs for instance $instanceId"
+
+# Collect the logs for the instance
+aws logs get-log-events --log-group-name /buildkite/buildkite-agent --log-stream-name "$instanceId" --output "$outputFormat" >> "$dir"/buildkite-agent-"$instanceId".log
+aws logs get-log-events --log-group-name /buildkite/system --log-stream-name "$instanceId" --output "$outputFormat" >> "$dir"/system-"$instanceId".log
+aws logs get-log-events --log-group-name /buildkite/lifecycled --log-stream-name "$instanceId" --output "$outputFormat" >> "$dir"/lifecycled-"$instanceId".log
+aws logs get-log-events --log-group-name /buildkite/docker-daemon --log-stream-name "$instanceId" --output "$outputFormat" >> "$dir"/docker-daemon-"$instanceId".log
+
+# CloudWatch stores the creationTime as milliseconds since epoch UTC. 
+# Since macOS doesn't support calculating the time since epoch in milliseconds, we don't have perfect accuracy on this, but it should be as close to 24 hours as possible
+# Currently only collects the last 24h of logs, but it might be good to be able to specify a different value for this.
+logAge=$(date -v-1d "+%s000")
+
+# Collect some information about the Lambda function and associated log groups
+# Change the timeframe to be dynamically set with a default of 24h. Currently collects the last 24 hours of Lambda logs
+lambdaFunction=$(aws cloudformation describe-stack-resources --stack-name "$stackName" --logical-resource-id Autoscaling --query "StackResources[*].PhysicalResourceId" --output text | sed -nr 's/.*stack\/(.*)\/.*/\1/p')
+lambdaLogGroup=$(aws cloudformation describe-stack-resources --stack-name "$lambdaFunction" --logical-resource-id LogGroup --query "StackResources[*].PhysicalResourceId" --output text)
+logStreams=$(aws logs describe-log-streams --log-group-name "$lambdaLogGroup" --order-by LastEventTime --query "logStreams[?creationTime > \`$logAge\`] | [*].logStreamName" --output text | tr -s '[:space:]' '\n')
+
+# Iterate through all of the log streams we collected from the lambda log group
+
+echo "Collecting Lambda logs for $lambdaFunction"
+for i in $logStreams
+do
+    fileName=$(echo "$i" | sed -r 's/\//-/g')
+    aws logs get-log-events --log-group-name "$lambdaLogGroup" --log-stream-name "$i" --output "$outputFormat" >> "$dir"/lambda-"$fileName".log
+done
+
+# Collect ASG activity for the scaling group
+autoScalingGroup=$(aws cloudformation describe-stack-resources --stack-name "$stackName" --logical-resource-id AgentAutoScaleGroup --query "StackResources[*].PhysicalResourceId" --output text)
+echo "Collecting ASG activity for $autoScalingGroup"
+aws autoscaling describe-scaling-activities --auto-scaling-group-name "$autoScalingGroup" >> "$dir"/asg-activities-"$stackName".log
+
+# Make an archive with the logs
+zip -jq buildkite-logs-"$now".zip "$dir"/*.log
+
+# Clean up the unneeded files
+rm -r "$dir"
+
+echo "Finished collecting logs, you can find them here: $pwd/buildkite-logs-$now.zip"
+
+exit 0