Merge branch 'master' of github.com:awslabs/cfncluster

Author: Dougal Ballantyne

CHANGELOG.rst

@@ -2,6 +2,12 @@
 CHANGELOG
 =========
 
+0.0.12
+======
+
+* updates:``cli``: Simplfied VPC config and removed multi-AZ
+
+
 0.0.11
 ======
 

README.md

@@ -8,16 +8,16 @@ everything is done using CloudFormation or resources within AWS.
 
 ### Installation
 
-The current working version is cfncluster-0.0.11. The CLI is written in python and uses BOTO for AWS actions. You can install the CLI with the following command:
+The current working version is cfncluster-0.0.12. The CLI is written in python and uses BOTO for AWS actions. You can install the CLI with the following command:
 
 #### Linux/OSX
 
 ```
-$ sudo easy_install cfncluster
+$ sudo pip install cfncluster
 ```
 or
 ```
-$ sudo pip install cfncluster
+$ sudo easy_install cfncluster
 ```
 
 #### Windows
@@ -31,6 +31,26 @@ setuptools - https://pypi.python.org/pypi/setuptools#windows-7-or-graphical-inst
 
 Once installed, you should update the Environment Variables to have the Python install directory and Python Scripts directory in the PATH, for example: C:\Python27;C:\Python27\Scripts
 
+Now it should be possible to run the following within a command prompt window:
+
+```
+C:\> easy_install cfncluster
+```
+
+#### Upgrading
+
+To upgrade an older version of cfncluster, you can use either of the following commands, depening on how it was originally installed:
+
+```
+$ sudo pip install --upgrade cfncluster
+```
+or
+```
+$ sudo easy_install -U cfncluster
+```
+
+** Remember when upgrading to check that the exiting config is compatible with the latest version installed.
+
 ### Configuration
 
 Once installed you will need to setup some initial config. The easiest way to do this is below:
@@ -45,30 +65,29 @@ $ mkdir ~/.cfncluster
 $ cp /usr/lib/python2.6/site-packages/cfncluster/examples/config ~/.cfncluster
 ```
 
-You should now edit the config and set some defaults before launching the cluster. First define a keypair that already exists in EC2.
+You should now edit the config and set some defaults before launching the cluster. First define a keypair that already exists in EC2. If you do not already have a keypair, refer to the EC2 documentation on EC2 Key Pairs - http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html
 
 ```
 [keypair mykey]
 key_location = /path/to/key.pem
+````
+
 Then you should associate that keypair with the cluster template.
+````
 [cluster default]
 # Name of an existing EC2 KeyPair to enable SSH access to the instances.
 key_name = mykey
 ```
 
-Finally, a base cluster launches into a VPC and uses an existing subnet which supports public IP's i.e. the route table for the subnet is 0.0.0.0/0 => igw-xxxxxx. The VPC must have "DNS Resolution = yes" and "DNS Hostnames = yes". It should also have DHCP options with the correct "domain-name" for the region, as defined in the docs: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html
+Next, a simple cluster launches into a VPC and uses an existing subnet which supports public IP's i.e. the route table for the subnet is 0.0.0.0/0 => igw-xxxxxx. The VPC must have "DNS Resolution = yes" and "DNS Hostnames = yes". It should also have DHCP options with the correct "domain-name" for the region, as defined in the docs: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html
 
 ```
 ## VPC Settings
 [vpc public]
 # ID of the VPC you want to provision cluster into.
-vpc_id = vpc-
+vpc_id = CHANGE ME, for example vpc-a1b2c3d4
 # ID of the Subnet you want to provision the Master server into
-public_subnet = subnet-
-# Availability zones of VPC resources
-# This is a comma delimited list and must always contain three values
-# Example: us-west-2a,NONE,NONE
-availability_zones =
+master_subnet_id = CHANGE ME, for exaple subnet-1ab2c3d4
 ```
 
 Once all of those settings contain valid values, you can launch the cluster by repeating the command that was used at the start.

bootstrap/src/scripts/boot_as_master

@@ -135,13 +135,13 @@ if [ $RC -ne 0 ]; then
   error_exit "Failed to start Ganglia"
 fi
 
-# Setup ec2-user SSH auth
+# Setup ${cfn_cluster_user} SSH auth
 RC=0
-su - ec2-user -c "ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ''" || RC=1
-su - ec2-user -c "cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys2 && chmod 0600 ~/.ssh/authorized_keys2" || RC=1
-su - ec2-user -c "ssh-keyscan ${myhostname} > ~/.ssh/known_hosts && chmod 0600 ~/.ssh/known_hosts" || RC=1
+su - ${cfn_cluster_user} -c "ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N ''" || RC=1
+su - ${cfn_cluster_user} -c "cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys2 && chmod 0600 ~/.ssh/authorized_keys2" || RC=1
+su - ${cfn_cluster_user} -c "ssh-keyscan ${myhostname} > ~/.ssh/known_hosts && chmod 0600 ~/.ssh/known_hosts" || RC=1
 if [ $RC -ne 0 ]; then
-  error_exit "Failed to setup ec2-user SSH auth"
+  error_exit "Failed to setup ${cfn_cluster_user} SSH auth"
 fi
 
 ## Scheduler specific section

cli/cfncluster/cfncluster.py

@@ -12,6 +12,7 @@
 import sys
 import boto.cloudformation
 import boto.ec2.autoscale
+import boto.vpc
 import boto.exception
 import time
 import os
@@ -25,14 +26,32 @@
 def create(args):
     print('Starting: %s' % (args.cluster_name))
 
+    # Build the config based on args
     config = cfnconfig.CfnClusterConfig(args)
+
+    # Set the ComputeWaitConditionCount parameter to match InitialQueueSize
     try:
         i = [p[0] for p in config.parameters].index('InitialQueueSize')
         initial_queue_size = config.parameters[i][1]
         config.parameters.append(('ComputeWaitConditionCount', initial_queue_size))
     except ValueError:
         pass
 
+    # Get the MasterSubnetId and use it to determine AvailabilityZone
+    try:
+        i = [p[0] for p in config.parameters].index('MasterSubnetId')
+        master_subnet_id = config.parameters[i][1]
+        try:
+            vpcconn = boto.vpc.connect_to_region(config.region,aws_access_key_id=config.aws_access_key_id,
+                                                 aws_secret_access_key=config.aws_secret_access_key)
+            availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone)
+        except boto.exception.BotoServerError as e:
+            print e.message
+            sys.exit(1)
+        config.parameters.append(('AvailabilityZone', availability_zone))
+    except ValueError:
+        pass
+
     capabilities = ["CAPABILITY_IAM"]
     cfnconn = boto.cloudformation.connect_to_region(config.region,aws_access_key_id=config.aws_access_key_id,
                                                     aws_secret_access_key=config.aws_secret_access_key)

cli/cfncluster/cfnconfig.py

@@ -117,25 +117,27 @@ def __init__(self, args):
             self.aws_secret_access_key=None
 
         # Determine the CloudFormation URL to be used
-        # Config file or default
+        # Order is 1) CLI arg 2) Config file 3) default for version + region
         try:
-            self.template_url = __config.get(self.__cluster_section,
-                                             'template_url')
-            if not self.template_url:
-                raise Exception
-        except ConfigParser.NoOptionError:
-            self.template_url = ('https://s3.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.version))
+            if args.template_url is not None:
+                self.template_url = args.template_url
+            try:
+                self.template_url = __config.get(self.__cluster_section,
+                                                 'template_url')
+                if not self.template_url:
+                    raise Exception
+            except ConfigParser.NoOptionError:
+                self.template_url = ('https://s3.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.version))
+        except AttributeError:
+            pass
 
         # Determine which vpc settings section will be used
         self.__vpc_settings = __config.get(self.__cluster_section, 'vpc_settings')
         self.__vpc_section = ('vpc %s' % self.__vpc_settings)
 
         # Dictionary list of all VPC options
-        self.__vpc_options = dict(vpc_id='VPCId', public_subnet='PublicSubnet', private_cidrs='PrivateCIDRs',
-                                  vpc_base_eni='VPCBaseNATENI1', compute_uses_public_subnet='ComputeUsesPublicSubnet',
-                                  vpc_base_security_group='VPCBaseBackSecurityGroup', use_vpc_base='UseVPCBase',
-                                  vpc_base_backend_subnet='VPCBaseBackendSubnet1',
-                                  availability_zones='AvailabilityZones', ssh_from='SSHFrom')
+        self.__vpc_options = dict(vpc_id='VPCId', master_subnet_id='MasterSubnetId', compute_subnet_cidr='ComputeSubnetCidr',
+                                  compute_subnet_id='ComputeSubnetId', use_public_ips='UsePublicIps' , ssh_from='SSHFrom')
 
         # Loop over all VPC options and add define to parameters, raise Exception is defined but null
         for key in self.__vpc_options:

cli/cfncluster/examples/config

@@ -99,33 +99,27 @@ vpc_settings = public
 # ID of the VPC you want to provision cluster into.
 vpc_id = vpc-
 # ID of the Subnet you want to provision the Master server into
-public_subnet = subnet-
-# Availability zones of VPC resources
-# This is a comma delimited list and must always contain three values
-# Example: us-west-2a,NONE,NONE
-availability_zones =
+master_subnet_id = subnet-
 # SSH from CIDR
 # This is only used when cfncluster creates the security group
 # (defaults to 0.0.0.0/0 in the default template)
 #ssh_from = 0.0.0.0/0
 
-#[vpc private]
-# Boolean flag to launch compute with direct egress or behind the Master server.
-#compute_uses_public_subnet = false
-# ID of the VPC you want to provision cluster into. Only used with UseVPCBase=false
-#vpc_id =
+#[vpc private-new]
+# ID of the VPC you want to provision cluster into.
+#vpc_id = vpc-
 # ID of the Subnet you want to provision the Master server into
-#public_subnet =
+#master_subnet_id = subnet-
 # CIDR for new backend subnet i.e. 10.0.100.0/24
-#private_cidrs = 10.0.100.0/24
+#compute_subnet_cidr = 10.0.100.0/24
 
-#[vpc existing-public]
-# Boolean flag to use exisiting vpc resources from vpc or create resources
-#use_vpc_base = true
-# Id of existing ENI to be used for the master serer
-#vpc_base_eni = eni-
-# Exisiting Compute Security Group ID
-#vpc_base_security_group = sg-
+#[vpc private-existing]
+# ID of the VPC you want to provision cluster into.
+#vpc_id =
+# ID of the Subnet you want to provision the Master server into
+#master_subnet_id = subnet-
+# CIDR for new backend subnet i.e. 10.0.100.0/24
+#compute_subnet_id = subnet-
 
 ## EBS Settings
 #[ebs custom]

cli/setup.py

@@ -20,8 +20,8 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 console_scripts = ['cfncluster = cfncluster.cli:main']
-version = "0.0.11"
-requires = ['boto', 'botocore'] 
+version = "0.0.12"
+requires = ['boto>=2.32.1', 'botocore'] 
 
 if sys.version_info[:2] == (2, 6):
     # For python2.6 we have to require argparse since it