auto expanding docker data volume fixes (#72)

* explicitly stop ecs before starting ebs autoscale on /var/lib/docker
* move nextflow additions to before start ecs
* fix autoexpanding docker data volume
  * add steps missing that are documented here: https://docs.docker.com/storage/storagedriver/btrfs-driver/
* fix scratch mount point mapping

Author: wleepang

src/templates/aws-genomics-launch-template.template.yaml

@@ -23,11 +23,14 @@ Mappings:
   ECSAdditionsMap:
     step-functions:
       additions: |-
+        - stop ecs
         - service docker stop
-        - cp -au /var/lib/docker /var/lib/docker.bk  
+        - cp -au /var/lib/docker /var/lib/docker.bk
+        - rm -rf /var/lib/docker/*
         - cd /opt && wget $artifactRootUrl/aws-ebs-autoscale.tgz && tar -xzf aws-ebs-autoscale.tgz
         - sh /opt/ebs-autoscale/bin/init-ebs-autoscale.sh $scratchPath /dev/sdc  2>&1 > /var/log/init-ebs-autoscale.log
         - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage
+        - cp -au /var/lib/docker.bk/* /var/lib/docker
         - service docker start
         - start ecs
 
@@ -40,15 +43,18 @@ Mappings:
 
     nextflow:
       additions: |-
+        - stop ecs
         - service docker stop
-        - cp -au /var/lib/docker /var/lib/docker.bk  
+        - cp -au /var/lib/docker /var/lib/docker.bk
+        - rm -rf /var/lib/docker/*
         - cd /opt && wget $artifactRootUrl/aws-ebs-autoscale.tgz && tar -xzf aws-ebs-autoscale.tgz
         - sh /opt/ebs-autoscale/bin/init-ebs-autoscale.sh $scratchPath /dev/sdc  2>&1 > /var/log/init-ebs-autoscale.log
         - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage
-        - service docker start
-        - start ecs
+        - cp -au /var/lib/docker.bk/* /var/lib/docker
         - cd /opt && wget $artifactRootUrl/aws-ecs-additions.tgz && tar -xzf aws-ecs-additions.tgz
         - sh /opt/ecs-additions/ecs-additions-nextflow.sh 
+        - service docker start
+        - start ecs
         
 Parameters:
   LaunchTemplateNamePrefix:
@@ -71,18 +77,12 @@ Parameters:
       - step-functions
       - cromwell
       - nextflow
-  ScratchMountPoint:
-    Type: String
-    Default: /scratch
-    Description: >-
-      Path for the scratch mount point in the instance. If your workflow 
-      orchestrator is "Cromwell" the value here is overridden with "/cromwell_root".
-      If your workflow orcheistrator is "Nextflow" the value here is overriden with "/var/lib/docker".
   ArtifactRootUrl:
     Type: String
     Default: https://s3.amazonaws.com/aws-genomics-workflows/artifacts
     Description: >-
       Root URL for where artifacts / additions scripts are stored
+
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -94,13 +94,9 @@ Metadata:
       - Label:
           default: "Optional"
         Parameters:
-          - ScratchMountPoint
           - ScratchVolumeSize
           - DockerImageAndMetadataVolumeSize
-Conditions:
-  UseCromwell: !Equals [!Ref WorkflowOrchestrator, cromwell]
-  UseNextflow: !Equals [!Ref WorkflowOrchestrator, nextflow]
-  ScratchOverride: !Or [Condition: UseCromwell, Condition: UseNextflow]
+
 Resources:
   EC2LaunchTemplate:
     Type: AWS::EC2::LaunchTemplate
@@ -160,11 +156,11 @@ Resources:
                   - ECSAdditionsMap
                   - !Ref WorkflowOrchestrator
                   - additions
-              ScratchMountPoint: 
-                Fn::If:
-                  - ScratchOverride
-                  - !FindInMap [ ScratchMountPointMap, !Ref WorkflowOrchestrator, mountpoint ]
-                  - !Ref ScratchMountPoint
+              ScratchMountPoint:
+                Fn::FindInMap:
+                  - ScratchMountPointMap
+                  - !Ref WorkflowOrchestrator
+                  - mountpoint
 
 Outputs:
   LaunchTemplateId: