Merge branch 'master' into add_swap_space

Author: wleepang

src/containers/nextflow/nextflow.aws.sh

@@ -52,7 +52,7 @@ cat ./nextflow.config
 
 # stage in session cache
 # .nextflow directory holds all session information for the current and past runs.
-# it should be `sync`'d with an s3 uri, so that runs from previous sessions can be 
+# it should be `sync`'d with an s3 uri, so that runs from previous sessions can be
 # resumed
 echo "== Restoring Session Cache =="
 aws s3 sync --no-progress $NF_LOGSDIR/.nextflow .nextflow

src/ecs-additions/awscli-shim.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This shim is for using the AWS ClI v2 with containers that do not have full glibc
+# it makes the shared libraries the AWS CLI v2 findable via LD_LIBRARY_PATH
+#
+# expect to be installed as /opt/aws-cli/bin/aws
+# expect to actually call /opt/aws-cli/dist/aws
+# expect that /opt/aws-cli is mapped to containers
+
+BIN_DIR=`dirname $0`
+DIST_DIR=`dirname $BIN_DIR`/dist
+AWS=$DIST_DIR/aws
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DIST_DIR
+
+$AWS $@
+

src/ecs-additions/ecs-additions-common.sh

@@ -4,10 +4,16 @@
 chmod a+x /opt/ecs-additions/fetch_and_run.sh
 cp /opt/ecs-additions/fetch_and_run.sh /usr/local/bin
 
+# add awscli-shim
+mv /opt/aws-cli/bin /opt/aws-cli/dist
+chmod a+x /opt/ecs-additions/awscli-shim.sh
+mkdir /opt/aws-cli/bin
+cp /opt/ecs-additions/awscli-shim.sh /opt/aws-cli/bin/aws
+
 # add 4GB of swap space
 dd if=/dev/zero of=/swapfile bs=128M count=32
 chmod 600 /swapfile
 mkswap /swapfile
 swapon /swapfile
 swapon -s
-echo '/swapfile swap swap defaults 0 0' >> /etc/fstab
+echo '/swapfile swap swap defaults 0 0' >> /etc/fstab

src/ecs-additions/fetch_and_run.sh

@@ -16,6 +16,7 @@
 # See below for usage instructions.
 
 #PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/aws-cli/v2/current/dist
 BASENAME="${0##*/}"
 AWS="/usr/local/aws-cli/v2/current/bin/aws"
 

src/templates/gwfcore/gwfcore-batch.template.yaml

@@ -128,6 +128,8 @@ Resources:
       Type: MANAGED
       State: ENABLED
       ComputeResources:
+        Ec2Configuration:
+          - ImageType: ECS_AL2
         AllocationStrategy: SPOT_CAPACITY_OPTIMIZED
         # Set the Spot price to 100% of on-demand price
         # This is the maximum price for spot instances that Batch will launch.
@@ -257,6 +259,12 @@ Resources:
         - Order: 2
           ComputeEnvironment: !Ref OnDemandComputeEnv
 
+  ContainerInstanceLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/ecs/container-instance/${Namespace}"
+      RetentionInDays: 7
+
 Outputs:
   DefaultJobQueueArn:
     Value: !Ref DefaultQueue

src/templates/gwfcore/gwfcore-iam.template.yaml

@@ -129,6 +129,7 @@ Resources:
       - "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
       - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
       - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+      - "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
   BatchInstanceProfile:
     Type: AWS::IAM::InstanceProfile
     Properties:

src/templates/gwfcore/gwfcore-launch-template.template.yaml

@@ -94,15 +94,72 @@ Resources:
                 --==BOUNDARY==
                 Content-Type: text/cloud-config; charset="us-ascii"
 
+                #cloud-config
+                repo_update: true
+                repo_upgrade: security
+
                 packages:
                 - jq
                 - btrfs-progs
                 - sed
                 - git
                 - amazon-ssm-agent
                 - unzip
+                - amazon-cloudwatch-agent
+
+                write_files:
+                - permissions: '0644'
+                  path: /opt/aws/amazon-cloudwatch-agent/etc/config.json
+                  content: |
+                    {
+                      "agent": {
+                        "logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
+                      },
+                      "logs": {
+                        "logs_collected": {
+                          "files": {
+                            "collect_list": [
+                              {
+                                "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/amazon-cloudwatch-agent.log"
+                              },
+                              {
+                                "file_path": "/var/log/cloud-init.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/cloud-init.log"
+                              },
+                              {
+                                "file_path": "/var/log/cloud-init-output.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/cloud-init-output.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-init.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-init.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-agent.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-agent.log"
+                              },
+                              {
+                                "file_path": "/var/log/ecs/ecs-volume-plugin.log",
+                                "log_group_name": "/aws/ecs/container-instance/${Namespace}",
+                                "log_stream_name": "/aws/ecs/container-instance/${Namespace}/{instance_id}/ecs-volume-plugin.log"
+                              }
+                            ]
+                          }
+                        }
+                      }
+                    }
 
                 runcmd:
+
+                # start the amazon-cloudwatch-agent
+                - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/config.json
+
                 # install aws-cli v2 and copy the static binary in an easy to find location for bind-mounts into containers
                 - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
                 - unzip -q /tmp/awscliv2.zip -d /tmp