Add ability to send certain logs that are written to while provisioning to Cloudwatch, to ease diagnosis of issue #114 or similar provisioning issues

Author: tthyer

src/templates/gwfcore/gwfcore-batch.template.yaml

@@ -257,6 +257,12 @@ Resources:
         - Order: 2
           ComputeEnvironment: !Ref OnDemandComputeEnv
 
+  ContainerInstanceLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: "/aws/ecs/container-instance"
+      RetentionInDays: 7
+
 Outputs:
   DefaultJobQueueArn:
     Value: !Ref DefaultQueue

src/templates/gwfcore/gwfcore-iam.template.yaml

@@ -129,6 +129,7 @@ Resources:
       - "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
       - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
       - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+      - "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
   BatchInstanceProfile:
     Type: AWS::IAM::InstanceProfile
     Properties:

src/templates/gwfcore/gwfcore-launch-template.template.yaml

@@ -94,15 +94,72 @@ Resources:
                 --==BOUNDARY==
                 Content-Type: text/cloud-config; charset="us-ascii"
 
+                #cloud-config
+                repo_update: true
+                repo_upgrade: all
+
                 packages:
                 - jq
                 - btrfs-progs
                 - sed
                 - git
                 - amazon-ssm-agent
                 - unzip
+                - amazon-cloudwatch-agent
 
                 runcmd:
+
+                # create the amazon-cloudwatch-agent config file
+                - |
+                  cat > /opt/aws/amazon-cloudwatch-agent/etc/config.json <<- EOF
+                  {
+                    "agent": {
+                      "logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
+                    },
+                    "logs": {
+                      "logs_collected": {
+                        "files": {
+                          "collect_list": [
+                            {
+                              "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/amazon-cloudwatch-agent.log"
+                            },
+                            {
+                              "file_path": "/var/log/cloud-init.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/cloud-init.log"
+                            },
+                            {
+                              "file_path": "/var/log/cloud-init-output.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/cloud-init-output.log"
+                            },
+                            {
+                              "file_path": "/var/log/ecs/ecs-init.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/ecs-init.log"
+                            },
+                            {
+                              "file_path": "/var/log/ecs/ecs-agent.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/ecs-agent.log"
+                            },
+                            {
+                              "file_path": "/var/log/ecs/ecs-volume-plugin.log",
+                              "log_group_name": "/aws/ecs/container-instance",
+                              "log_stream_name": "/aws/ecs/container-instance/{instance_id}/ecs-volume-plugin.log"
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  }
+                  EOF
+
+                # start the amazon-cloudwatch-agent
+                - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/config.json
+
                 # install aws-cli v2 and copy the static binary in an easy to find location for bind-mounts into containers
                 - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
                 - unzip -q /tmp/awscliv2.zip -d /tmp