Merge pull request #811 from buildkite/cloudwatch-agent

Replace awslogs with the cloudwatch-agent

Author: James Healy

goss.yaml

@@ -36,7 +36,7 @@ port:
     - '::'
 
 service:
-  awslogsd:
+  amazon-cloudwatch-agent:
     enabled: true
     running: true
 

packer/linux/buildkite-ami.json

@@ -48,7 +48,7 @@
     },
     {
       "type": "shell",
-      "script": "scripts/install-awslogs.sh"
+      "script": "scripts/install-cloudwatch-agent.sh"
     },
     {
       "type": "shell",

packer/linux/conf/awslogs/awslogs.conf

@@ -31,20 +31,6 @@ trap 'on_error $LINENO' ERR
 INSTANCE_ID=$(/opt/aws/bin/ec2-metadata --instance-id | cut -d " " -f 2)
 DOCKER_VERSION=$(docker --version | cut -f3 -d' ' | sed 's/,//')
 
-# Cloudwatch logs needs a region specifically configured
-cat << EOF > /etc/awslogs/awscli.conf
-[plugins]
-cwlogs = cwlogs
-[default]
-region = $AWS_REGION
-EOF
-
-systemctl enable awslogsd.service
-
-# Start logging daemons as soon as possible to ensure failures in this script get sent
-systemctl restart rsyslog
-systemctl restart awslogsd
-
 PLUGINS_ENABLED=()
 [[ $SECRETS_PLUGIN_ENABLED == "true" ]] && PLUGINS_ENABLED+=("secrets")
 [[ $ECR_PLUGIN_ENABLED == "true" ]] && PLUGINS_ENABLED+=("ecr")

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -0,0 +1,55 @@
+{
+	"agent": {
+		"run_as_user": "root"
+	},
+	"logs": {
+		"logs_collected": {
+			"files": {
+				"collect_list": [
+					{
+						"file_path": "/var/log/buildkite-agent",
+						"log_group_name": "/buildkite/buildkite-agent",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%b %d %H:%M:%S"
+					},
+					{
+						"file_path": "/var/log/cfn-init.log",
+						"log_group_name": "/buildkite/cfn-init",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+					},
+					{
+						"file_path": "/var/log/cloud-init.log",
+						"log_group_name": "/buildkite/cloud-init",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+					},
+					{
+						"file_path": "/var/log/cloud-init-output.log",
+						"log_group_name": "/buildkite/cloud-init/output",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+					},
+					{
+						"file_path": "/var/log/docker",
+						"log_group_name": "/buildkite/docker-daemon",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
+					},
+					{
+						"file_path": "/var/log/elastic-stack.log",
+						"log_group_name": "/buildkite/elastic-stack",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+					},
+					{
+						"file_path": "/var/log/messages",
+						"log_group_name": "/buildkite/system",
+						"log_stream_name": "{instance_id}",
+						"timestamp_format": "%b %d %H:%M:%S"
+					}
+				]
+			}
+		}
+	}
+}

packer/linux/conf/cloudwatch-agent/config.json

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+
+echo "Installing cloudwatch agent..."
+sudo yum install -y amazon-cloudwatch-agent
+
+echo "Adding amazon-cloudwatch-agent config..."
+sudo cp /tmp/conf/cloudwatch-agent/config.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+
+echo "Configuring amazon-cloudwatch-agent to start at boot"
+sudo systemctl enable amazon-cloudwatch-agent
+
+# These will send some systemctl service logs (like the buildkite agent and docker) to logfiles
+echo "Adding rsyslogd configs..."
+sudo cp /tmp/conf/cloudwatch-agent/rsyslog.d/* /etc/rsyslog.d/

packer/linux/conf/awslogs/rsyslog.d/buildkite-agent.conf renamed to packer/linux/conf/cloudwatch-agent/rsyslog.d/buildkite-agent.conf

@@ -47,7 +47,7 @@
     },
     {
       "type": "powershell",
-      "script": "scripts/install-awslogs.ps1"
+      "script": "scripts/install-cloudwatch-agent.ps1"
     },
     {
       "type": "powershell",