Merge pull request #127 from tthyer/misc-cloudwatch-logging

Ship logs to cloudwatch to diagnose provisioning errors

Author: wleepang

src/templates/gwfcore/gwfcore-batch.template.yaml

@@ -128,6 +128,8 @@ Resources:
       Type: MANAGED
       State: ENABLED
       ComputeResources:
+        Ec2Configuration:
+          - ImageType: ECS_AL2
         AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
         # Set the Spot price to 100% of on-demand price
         # This is the maximum price for spot instances that Batch will launch.
@@ -257,6 +259,12 @@ Resources:
         - Order: 2
           ComputeEnvironment: !Ref OnDemandComputeEnv
 
+  ContainerInstanceLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/ecs/container-instance/${Namespace}"
+      RetentionInDays: 7
+
 Outputs:
   DefaultJobQueueArn:
     Value: !Ref DefaultQueue

src/templates/gwfcore/gwfcore-iam.template.yaml

@@ -129,6 +129,7 @@ Resources:
       - "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
       - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
       - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+      - "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
   BatchInstanceProfile:
     Type: AWS::IAM::InstanceProfile
     Properties:

src/templates/gwfcore/gwfcore-launch-template.template.yaml

@@ -94,15 +94,72 @@ Resources:
                 --==BOUNDARY==
                 Content-Type: text/cloud-config; charset="us-ascii"
 
+                #cloud-config
+                repo_update: true
+                repo_upgrade: security
+
                 packages:
                 - jq
                 - btrfs-progs
                 - sed
                 - git
                 - amazon-ssm-agent
                 - unzip
+                - amazon-cloudwatch-agent
+
+                write_files:
+                - permissions: '0644'
+                  path: /opt/aws/amazon-cloudwatch-agent/etc/config.json
+                  content: |
+                    {
+                      "agent": {
+                        "logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
+                      },
+                      "logs": {
+                        "logs_collected": {
+                          "files": {
+                            "collect_list": [
+                              {
+                                "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/amazon-cloudwatch-agent.log"
+                              },
+                              {
+                                "file_path": "/var/log/cloud-init.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/cloud-init.log"
+                              },
+                              {
+                                "file_path": "/var/log/cloud-init-output.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/cloud-init-output.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-init.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-init.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-agent.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-agent.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-volume-plugin.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-volume-plugin.log"
+                              }
+                            ]
+                          }
+                        }
+                      }
+                    }
 
                 runcmd:
+
+                # start the amazon-cloudwatch-agent
+                - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/config.json
+
                 # install aws-cli v2 and copy the static binary in an easy to find location for bind-mounts into containers
                 - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
                 - unzip -q /tmp/awscliv2.zip -d /tmp