Merge pull request #194 from awslabs/develop

Merge 1.3.2 release into master

Author: bwbarrett

.travis.yml

@@ -6,6 +6,7 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
 
 install: 
   - cd cli

CHANGELOG.rst

@@ -2,6 +2,18 @@
 CHANGELOG
 =========
 
+1.3.2
+=====
+* feature:``cfncluster``: Add support for r2.xlarge/t2.2xlarge,
+  x1.16xlarge, r4.*, f1.*, and i3.* instance types
+* bugfix:``cfncluster``: Fix support for p2.2xlarge instance type
+* feature:``cfncluster``: Add support for eu-west-2, us-east-2, and
+  ca-central-1 regions
+* updates:``cfncluster-cookbook``: Updated to cfncluster-cookbook-1.3.2
+* updates:``ami``: Update all base AMIs to latest patch levels
+* updates:``cfncluster``: Moved to Apache 2.0 license
+* updates:``cfncluster``: Support for Python 3
+
 1.3.1
 =====
 * feature:``ami``: Added support for Ubuntu 16.04 LTS
@@ -198,4 +210,4 @@ changes should be non-instrusive.
 0.0.5
 =====
 
-* Release on GitHub and PyPi
+* Release on GitHub and PyPi

CONTRIBUTORS.txt

@@ -3,6 +3,7 @@ List of contributors:
  - Dougal Ballantyne - dougalb at amazon dot com
  - Adam Boeglin - boeglina at amazon dot com
  - Sean Smith - sensmit at amazon dot com
+ - Brian Barrett - bbarrett at amazon dot com
 
  - Kenneth Daily - kmdaily at gmail dot com
  - John Lilley - johnbot at ltech dot edu

README.rst

@@ -7,6 +7,15 @@ CfnCluster
 
 CfnCluster ("cloud formation cluster") is a framework that deploys and maintains high performance computing clusters on Amazon Web Services (AWS). Developed by AWS, CfnCluster facilitates both quick start proof of concepts (POCs) and production deployments. CfnCluster supports many different types of clustered applications and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS.
 
+Known Issues
+============
+
+* CfnCluster 1.3.2 supports the I3 and F1 instance families.  However,
+  it will not automatically setup a single, RAIDed volume across all
+  available instance store volumes (like it will for other instance
+  families which support instance storage).  This will be fixed in a
+  future release.
+
 Documentation
 =============
 
@@ -15,4 +24,6 @@ Documentation is part of the project and is published to - https://cfncluster.re
 Issues
 ======
 
-Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192
+Please open a GitHub issue for any feedback or issues:
+https://github.com/awslabs/cfncluster.  There is also an active AWS
+HPC forum which may be helpful:https://forums.aws.amazon.com/forum.jspa?forumID=192.

amis.txt

@@ -1,60 +1,75 @@
 # centos6
-ap-northeast-1: ami-46b17b27
-ap-northeast-2: ami-75e1341b
-ap-south-1: ami-a392e7cc
-ap-southeast-1: ami-23f22940
-ap-southeast-2: ami-0ea6906d
-eu-central-1: ami-5fd92a30
-eu-west-1: ami-27f78554
-sa-east-1: ami-8ad848e6
-us-east-1: ami-504b2047
-us-west-1: ami-051f5365
-us-west-2: ami-6e30e10e
+ap-northeast-1: ami-10686677
+ap-northeast-2: ami-bb77abd5
+ap-south-1: ami-0a710e65
+ap-southeast-1: ami-9b1596f8
+ap-southeast-2: ami-c7b9afa4
+ca-central-1: ami-a6f847c2
+eu-central-1: ami-56d67239
+eu-west-1: ami-abd5c9cd
+eu-west-2: ami-cabcabae
+sa-east-1: ami-b491f9d8
+us-east-1: ami-a293c9b4
+us-east-2: ami-2cd3f549
+us-west-1: ami-65c2e005
+us-west-2: ami-6bb1b812
 # centos7
-ap-northeast-1: ami-43c90322
-ap-northeast-2: ami-5cec3932
-ap-south-1: ami-bf90e5d0
-ap-southeast-1: ami-d5f52eb6
-ap-southeast-2: ami-79a5931a
-eu-central-1: ami-fcda2993
-eu-west-1: ami-5af48629
-sa-east-1: ami-f8d84894
-us-east-1: ami-e94328fe
-us-west-1: ami-a51955c5
-us-west-2: ami-2e1dcc4e
+ap-northeast-1: ami-0e6b6569
+ap-northeast-2: ami-9377abfd
+ap-south-1: ami-546d123b
+ap-southeast-1: ami-da1596b9
+ap-southeast-2: ami-afa6b0cc
+ca-central-1: ami-62f94606
+eu-central-1: ami-9cd571f3
+eu-west-1: ami-6dd8c40b
+eu-west-2: ami-9cb9aef8
+sa-east-1: ami-5092fa3c
+us-east-1: ami-d29cc6c4
+us-east-2: ami-c3d2f4a6
+us-west-1: ami-17fcde77
+us-west-2: ami-22b2bb5b
 # alinux
-ap-northeast-1: ami-50bd7731
-ap-northeast-2: ami-afe431c1
-ap-south-1: ami-0495e06b
-ap-southeast-1: ami-9af229f9
-ap-southeast-2: ami-51b98f32
-eu-central-1: ami-96db28f9
-eu-west-1: ami-d8f785ab
-sa-east-1: ami-8bd848e7
-us-east-1: ami-d5751ec2
-us-west-1: ami-5f19553f
-us-west-2: ami-1e31e07e
+ap-northeast-1: ami-586b653f
+ap-northeast-2: ami-ea71ad84
+ap-south-1: ami-1f720d70
+ap-southeast-1: ami-1f1b987c
+ap-southeast-2: ami-56b9af35
+ca-central-1: ami-65f94601
+eu-central-1: ami-7fd17510
+eu-west-1: ami-6ddac60b
+eu-west-2: ami-f1bbac95
+sa-east-1: ami-a990f8c5
+us-east-1: ami-2492c832
+us-east-2: ami-a6d3f5c3
+us-west-1: ami-39c2e059
+us-west-2: ami-f4b0b98d
 # ubuntu1404
-ap-northeast-1: ami-89bd77e8
-ap-northeast-2: ami-bae732d4
-ap-south-1: ami-489de827
-ap-southeast-1: ami-95f229f6
-ap-southeast-2: ami-d3b98fb0
-eu-central-1: ami-89db28e6
-eu-west-1: ami-3ff0824c
-sa-east-1: ami-3f34a553
-us-east-1: ami-ba4a21ad
-us-west-1: ami-40185420
-us-west-2: ami-4633e226
+ap-northeast-1: ami-466c6221
+ap-northeast-2: ami-2877ab46
+ap-south-1: ami-11700f7e
+ap-southeast-1: ami-be1497dd
+ap-southeast-2: ami-37a4b254
+ca-central-1: ami-eefc438a
+eu-central-1: ami-bcd773d3
+eu-west-1: ami-83d4c8e5
+eu-west-2: ami-5cb9ae38
+sa-east-1: ami-6d91f901
+us-east-1: ami-a192c8b7
+us-east-2: ami-e2d3f587
+us-west-1: ami-b5c2e0d5
+us-west-2: ami-f5b0b98c
 # ubuntu1604
-ap-northeast-1: ami-eabe748b
-ap-northeast-2: ami-7eef3a10
-ap-south-1: ami-5d90e532
-ap-southeast-1: ami-23f32840
-ap-southeast-2: ami-c1b98fa2
-eu-central-1: ami-9dd427f2
-eu-west-1: ami-99f082ea
-sa-east-1: ami-8d0e9fe1
-us-east-1: ami-d6751ec1
-us-west-1: ami-751f5315
-us-west-2: ami-870edfe7
+ap-northeast-1: ami-ce6c62a9
+ap-northeast-2: ami-d072aebe
+ap-south-1: ami-446f102b
+ap-southeast-1: ami-951596f6
+ap-southeast-2: ami-cca6b0af
+ca-central-1: ami-a7fa45c3
+eu-central-1: ami-40d6722f
+eu-west-1: ami-e6d5c980
+eu-west-2: ami-5eb9ae3a
+sa-east-1: ami-df90f8b3
+us-east-1: ami-8c9fc59a
+us-east-2: ami-54d5f331
+us-west-1: ami-0dfddf6d
+us-west-2: ami-c4b1b8bd

cli/cfncluster/cfncluster.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+from __future__ import absolute_import
 # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
@@ -9,6 +11,7 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import str
 import sys
 import boto.cloudformation
 import boto.ec2.autoscale
@@ -19,13 +22,13 @@
 import socket
 import logging
 
-import cfnconfig
+from . import cfnconfig
 
 logger = logging.getLogger('cfncluster.cfncluster')
 
 def version(args):
     config = cfnconfig.CfnClusterConfig(args)
-    print config.version
+    print(config.version)
 
 def create(args):
     print('Starting: %s' % (args.cluster_name))
@@ -50,7 +53,7 @@ def create(args):
                                                  aws_secret_access_key=config.aws_secret_access_key)
             availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone)
         except boto.exception.BotoServerError as e:
-            print e.message
+            print(e.message)
             sys.exit(1)
         config.parameters.append(('AvailabilityZone', availability_zone))
     except ValueError:
@@ -75,12 +78,12 @@ def create(args):
                 time.sleep(5)
             outputs = cfnconn.describe_stacks(stack)[0].outputs
             for output in outputs:
-                print output
+                print(output)
         else:
             status = cfnconn.describe_stacks(stack)[0].stack_status
             print('Status: %s' % status)
     except boto.exception.BotoServerError as e:
-        print e.message
+        print(e.message)
         sys.exit(1)
     except KeyboardInterrupt:
         print('\nExiting...')
@@ -118,7 +121,7 @@ def update(args):
                                                  aws_secret_access_key=config.aws_secret_access_key)
             availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone)
         except boto.exception.BotoServerError as e:
-            print e.message
+            print(e.message)
             sys.exit(1)
         config.parameters.append(('AvailabilityZone', availability_zone))
     except ValueError:
@@ -142,7 +145,7 @@ def update(args):
             status = cfnconn.describe_stacks(stack)[0].stack_status
             print('Status: %s' % status)
     except boto.exception.BotoServerError as e:
-        print e.message
+        print(e.message)
         sys.exit(1)
     except KeyboardInterrupt:
         print('\nExiting...')
@@ -161,7 +164,7 @@ def start(args):
         response = ec2conn.start_instances(master_server_id)
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else:
@@ -173,11 +176,13 @@ def start(args):
     # Set asg limits
     max_queue_size = [param[1] for param in config.parameters if param[0] == 'MaxQueueSize']
     max_queue_size = max_queue_size[0] if len(max_queue_size) > 0 else 10
-    initial_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] if args.reset_desired else [0]
-    initial_queue_size = initial_queue_size[0] if len(initial_queue_size) > 0 else 0
+    desired_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize']
+    desired_queue_size = desired_queue_size[0] if len(desired_queue_size) > 0 else 2
+    min_queue_size = [desired_queue_size for param in config.parameters if param[0] == 'MaintainInitialSize' and param[1] == "true"]
+    min_queue_size = min_queue_size[0] if len(min_queue_size) > 0 else 0
 
     asg = get_asg(stack_name=stack_name, config=config)
-    set_asg_limits(asg=asg, min=initial_queue_size, max=max_queue_size, desired=initial_queue_size)
+    set_asg_limits(asg=asg, min=min_queue_size, max=max_queue_size, desired=desired_queue_size)
 
     # Poll for status
     poll_master_server_state(stack_name, config)
@@ -201,7 +206,7 @@ def stop(args):
         response = ec2conn.stop_instances(master_server_id)
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else:
@@ -224,7 +229,7 @@ def list(args):
                 print('%s' % (stack.stack_name[11:]))
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
-            print e.message
+            print(e.message)
         else:
             raise e
     except KeyboardInterrupt:
@@ -243,7 +248,7 @@ def get_master_server_id(stack_name, config):
             resources = cfnconn.describe_stack_resources(stack_name)
         except boto.exception.BotoServerError as e:
             if e.message.endswith("does not exist"):
-                print e.message
+                print(e.message)
                 sys.stdout.flush()
                 sys.exit(0)
             else:
@@ -280,7 +285,7 @@ def poll_master_server_state(stack_name, config):
         sys.stdout.flush()
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else:
@@ -303,7 +308,7 @@ def get_ec2_instances(stack, config):
         except boto.exception.BotoServerError as e:
             if e.message.endswith("does not exist"):
                 #sys.stdout.write('\r\n')
-                print e.message
+                print(e.message)
                 sys.stdout.flush()
                 sys.exit(0)
             else:
@@ -334,7 +339,7 @@ def get_asg(stack_name, config):
         return asgconn.get_all_groups(names=[asg_id])[0]
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else:
@@ -363,7 +368,7 @@ def get_asg_ids(stack, config):
         except boto.exception.BotoServerError as e:
             if e.message.endswith("does not exist"):
                 #sys.stdout.write('\r\n')
-                print e.message
+                print(e.message)
                 sys.stdout.flush()
                 sys.exit(0)
             else:
@@ -433,22 +438,22 @@ def status(args):
                 if state == 'running':
                     outputs = cfnconn.describe_stacks(stack)[0].outputs
                     for output in outputs:
-                        print output
+                        print(output)
             elif ((status == 'ROLLBACK_COMPLETE') or (status == 'CREATE_FAILED') or (status == 'DELETE_FAILED') or
                       (status == 'UPDATE_ROLLBACK_COMPLETE')):
                 events = cfnconn.describe_stack_events(stack)
                 for event in events:
                     if ((event.resource_status == 'CREATE_FAILED') or (event.resource_status == 'DELETE_FAILED') or
                             (event.resource_status == 'UPDATE_FAILED')):
-                        print event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \
-                            event.resource_status_reason
+                        print(event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \
+                            event.resource_status_reason)
         else:
             sys.stdout.write('\n')
             sys.stdout.flush()
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
             sys.stdout.write('\r')
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else:
@@ -489,7 +494,7 @@ def delete(args):
     except boto.exception.BotoServerError as e:
         if e.message.endswith("does not exist"):
             #sys.stdout.write('\r\n')
-            print e.message
+            print(e.message)
             sys.stdout.flush()
             sys.exit(0)
         else: