PR from devlop for release v0.0.19

Author: dougalb

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+0.0.19
+======
+* feature:``cli``: Added configure command; easy config setup
+* updates:``docs``: Addtional documentation for configuration options
+* updates:``ami``: Pulled latest CentOS6 errata
+* bugfix:``cfncluster``: Fixed issue with nodewatcher not scaling down
+
 0.0.18
 ======
 * updates:``ami``: Custom CentOS 6 kernel repo added, support for >32 vCPUs

README.rst

@@ -7,3 +7,8 @@ Documentation
 =============
 
 Documentation is part of the project and is published to - http://cfncluster.readthedocs.org/. Of most interest to new users is the Getting Started Guide - http://cfncluster.readthedocs.org/en/latest/getting_started.html.
+
+Issues
+======
+
+Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192

amis.txt

@@ -1,10 +1,10 @@
-us-west-2 ami-87de82b7
-eu-central-1 ami-8c97a791
-sa-east-1 ami-2bdd6036
-ap-northeast-1 ami-a8d8c8a9
-eu-west-1 ami-f738be80
-us-east-1 ami-4ad6a522
-us-west-1 ami-298e916c
-ap-southeast-2 ami-efc4aed5
-ap-southeast-1 ami-705a7222
-us-gov-west-1 ami-5fcfa97c
+us-west-2 ami-ab7f599b
+eu-central-1 ami-b43705a9
+sa-east-1 ami-818b359c
+ap-northeast-1 ami-891bfb89
+eu-west-1 ami-87fd69f0
+us-east-1 ami-7660361e
+us-west-1 ami-86b0aac3
+ap-southeast-2 ami-7d730547
+ap-southeast-1 ami-fafcc8a8
+us-gov-west-1 ami-5d68097e

bootstrap/src/scripts/boot_as_compute

@@ -15,6 +15,7 @@ set -x
 
 # Source config
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 # Source functions
 . /opt/cfncluster/scripts/functions.shlib

bootstrap/src/scripts/openlava/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 function error_exit () {
   script=`basename $0`

bootstrap/src/scripts/sge/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 # Source functions
 . /opt/cfncluster/scripts/functions.shlib

bootstrap/src/scripts/torque/boot_as_compute

@@ -14,6 +14,7 @@
 set -x
 
 . /opt/cfncluster/cfnconfig
+cfn_master=$(echo $cfn_master|cut -d. -f1)
 
 function error_exit () {
   script=`basename $0`

cli/cfncluster/cfnconfig.py

@@ -14,10 +14,20 @@
 import sys
 import inspect
 import pkg_resources
-import logging
 import json
 import urllib2
 import config_sanity
+import boto.cloudformation
+
+def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack):
+
+    cfn_conn = boto.cloudformation.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+    __stack_name = ('cfncluster-' + stack)
+    __stack = cfn_conn.describe_stacks(stack_name_or_id=__stack_name)[0]
+    __cli_template = [p.value for p in __stack.parameters if p.key == 'CLITemplate'][0]
+
+    return __cli_template
 
 class CfnClusterConfig:
 
@@ -26,6 +36,7 @@ def __init__(self, args):
         self.parameters = []
         self.version = pkg_resources.get_distribution("cfncluster").version
         self.__DEFAULT_CONFIG = False
+        __args_func = self.args.func.func_name
 
         # Determine config file name based on args or default
         if args.config_file is not None:
@@ -50,12 +61,39 @@ def __init__(self, args):
         __config = ConfigParser.ConfigParser()
         __config.read(self.__config_file)
 
+        # Determine the EC2 region to used used or default to us-east-1
+        # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
+        if args.region:
+            self.region = args.region
+        else:
+            if os.environ.get('AWS_DEFAULT_REGION'):
+                self.region = os.environ.get('AWS_DEFAULT_REGION')
+            else:
+                try:
+                    self.region = __config.get('aws', 'aws_region_name')
+                except ConfigParser.NoOptionError:
+                    self.region = 'us-east-1'
+
+        # Check if credentials have been provided in config
+        try:
+            self.aws_access_key_id = __config.get('aws', 'aws_access_key_id')
+        except ConfigParser.NoOptionError:
+            self.aws_access_key_id=None
+        try:
+            self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key')
+        except ConfigParser.NoOptionError:
+            self.aws_secret_access_key=None
+
         # Determine which cluster template will be used
         try:
             if args.cluster_template is not None:
                 self.__cluster_template = args.cluster_template
             else:
-                self.__cluster_template = __config.get('global', 'cluster_template')
+                if __args_func == 'update':
+                    self.__cluster_template = getStackTemplate(self.region,self.aws_access_key_id,
+                                                               self.aws_secret_access_key, self.args.cluster_name)
+                else:
+                    self.__cluster_template = __config.get('global', 'cluster_template')
         except AttributeError:
             self.__cluster_template = __config.get('global', 'cluster_template')
         self.__cluster_section = ('cluster %s' % self.__cluster_template)
@@ -82,34 +120,11 @@ def __init__(self, args):
             self.__sanity_check = False
         # Only check config on calls that mutate it
         __args_func = self.args.func.func_name
-        if (__args_func == 'create' or __args_func == 'update') and self.__sanity_check is True:
+        if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True:
             pass
         else:
             self.__sanity_check = False
 
-        # Determine the EC2 region to used used or default to us-east-1
-        # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
-        if args.region:
-            self.region = args.region
-        else:
-            if os.environ.get('AWS_DEFAULT_REGION'):
-                self.region = os.environ.get('AWS_DEFAULT_REGION')
-            else:
-                try:
-                    self.region = __config.get('aws', 'aws_region_name')
-                except ConfigParser.NoOptionError:
-                    self.region = 'us-east-1'
-
-        # Check if credentials have been provided in config
-        try:
-            self.aws_access_key_id = __config.get('aws', 'aws_access_key_id')
-        except ConfigParser.NoOptionError:
-            self.aws_access_key_id=None
-        try:
-            self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key')
-        except ConfigParser.NoOptionError:
-            self.aws_secret_access_key=None
-
         # Get the EC2 keypair name to be used, exit if not set
         try:
             self.key_name = __config.get(self.__cluster_section, 'key_name')

cli/cfncluster/cli.py

@@ -16,10 +16,14 @@
 import json
 
 import cfncluster
+import easyconfig
 
 def create(args):
     cfncluster.create(args)
 
+def configure(args):
+    easyconfig.configure(args)
+
 def status(args):
     cfncluster.status(args)
 
@@ -121,9 +125,12 @@ def main():
                         help='show the status of cfncluster with the provided name.')
     pinstances.set_defaults(func=instances)
 
+    pconfigure = subparsers.add_parser('configure', help='creating initial cfncluster configuration')
+    pconfigure.set_defaults(func=configure)
+
     pversion = subparsers.add_parser('version', help='display version of cfncluster')
     pversion.set_defaults(func=version)
 
     args = parser.parse_args()
     logging.debug(args)
-    args.func(args)
+    args.func(args)

cli/cfncluster/config_sanity.py

@@ -39,6 +39,13 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty
         except boto.exception.BotoServerError as e:
             print('Config sanity error: %s' % e.message)
             sys.exit(1)
+        # Check for DNS support in the VPC
+        if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsSupport').enable_dns_support:
+            print("DNS Support is not enabled in %s" % test[0].id)
+            sys.exit(1)
+        if not vpc_conn.describe_vpc_attribute(test[0].id, attribute='enableDnsHostnames').enable_dns_hostnames:
+            print("DNS Hostnames not enabled in %s" % test[0].id)
+            sys.exit(1)
     # VPC Subnet Id
     elif resource_type == 'VPCSubnet':
         try: