New sanity check for CLI; multi-OS changes; move to supervisord for daemons

Author: Dougal Ballantyne

bootstrap/src/scripts/boot_as_compute

@@ -54,14 +54,9 @@ if [ $RC -ne 0 ]; then
   error_exit "Failed during Ganglia setup"
 fi
 
-# Adding nodewatcher to crontab
-RC=0
-crontab -l > /tmp/root.crontab
-echo "* * * * * cd /opt/cfncluster/nodewatcher && ./nodewatcher.py >> nodewatcher.log 2>&1" >> /tmp/root.crontab || RC=1
-crontab /tmp/root.crontab || RC=1
-if [ $RC -ne 0 ]; then
-  error_exit "Failed to nodewatcher crontab"
-fi
+# Startup nodewatcher
+service supervisord restart
+supervisorctl status
 
 ##
 # Run boot as compute for a specific scheduler

bootstrap/src/scripts/boot_as_master

@@ -155,7 +155,8 @@ if [ $RC -ne 0 ]; then
 fi
 
 # Start sqswatcher
-cd /opt/cfncluster/sqswatcher && ./sqswatcher.py 2>&1
+service supervisord restart
+supervisorctl status
 
 # Run postinstall script if defined
 run_postinstall

cli/cfncluster/cfnconfig.py

@@ -17,6 +17,7 @@
 import logging
 import json
 import urllib2
+import config_sanity
 
 class CfnClusterConfig:
 
@@ -73,25 +74,11 @@ def __init__(self, args):
             except Exception:
                 pass
 
-        # Get the EC2 keypair name to be used, exit if not set
+        # Check if config sanity should be run
         try:
-            self.key_name = __config.get(self.__cluster_section, 'key_name')
-            if not self.key_name:
-                raise Exception
+            self.__sanity_check = __config.getboolean('global', 'sanity_check')
         except ConfigParser.NoOptionError:
-            raise Exception
-        self.parameters.append(('KeyName', self.key_name))
-
-        # Determine which keypair config will be used
-        self.__keypair_section = ('keypair %s' % self.key_name)
-
-        # Get the location of the keypair file
-        try:
-            self.key_location = __config.get(self.__keypair_section, 'key_location')
-            if not self.key_location:
-                raise Exception
-        except ConfigParser.NoOptionError:
-            raise Exception
+            self.__sanity_check = False
 
         # Determine the EC2 region to used used or default to us-east-1
         # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
@@ -116,6 +103,18 @@ def __init__(self, args):
         except ConfigParser.NoOptionError:
             self.aws_secret_access_key=None
 
+        # Get the EC2 keypair name to be used, exit if not set
+        try:
+            self.key_name = __config.get(self.__cluster_section, 'key_name')
+            if not self.key_name:
+                raise Exception
+            if self.__sanity_check:
+                config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                             'EC2KeyPair', self.key_name)
+        except ConfigParser.NoOptionError:
+            raise Exception
+        self.parameters.append(('KeyName', self.key_name))
+
         # Determine the CloudFormation URL to be used
         # Order is 1) CLI arg 2) Config file 3) default for version + region
         try:
@@ -126,6 +125,9 @@ def __init__(self, args):
                                                  'template_url')
                 if not self.template_url:
                     raise Exception
+                if self.__sanity_check:
+                    config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                             'URL', self.template_url)
             except ConfigParser.NoOptionError:
                 self.template_url = ('https://s3.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.version))
         except AttributeError:
@@ -136,38 +138,46 @@ def __init__(self, args):
         self.__vpc_section = ('vpc %s' % self.__vpc_settings)
 
         # Dictionary list of all VPC options
-        self.__vpc_options = dict(vpc_id='VPCId', master_subnet_id='MasterSubnetId', compute_subnet_cidr='ComputeSubnetCidr',
-                                  compute_subnet_id='ComputeSubnetId', use_public_ips='UsePublicIps' , ssh_from='SSHFrom')
+        self.__vpc_options = dict(vpc_id=('VPCId','VPC'), master_subnet_id=('MasterSubnetId', 'VPCSubnet'),
+                                  compute_subnet_cidr=('ComputeSubnetCidr',None),
+                                  compute_subnet_id=('ComputeSubnetId', 'VPCSubnet'), use_public_ips=('UsePublicIps', None),
+                                  ssh_from=('SSHFrom', None))
 
         # Loop over all VPC options and add define to parameters, raise Exception is defined but null
         for key in self.__vpc_options:
             try:
                 __temp__ = __config.get(self.__vpc_section, key)
                 if not __temp__:
                     raise Exception
-                self.parameters.append((self.__vpc_options.get(key),__temp__))
+                if self.__sanity_check and self.__vpc_options.get(key)[1] is not None:
+                    config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                                self.__vpc_options.get(key)[1],__temp__)
+                self.parameters.append((self.__vpc_options.get(key)[0],__temp__))
             except ConfigParser.NoOptionError:
                 pass
 
         # Dictionary list of all cluster section options
-        self.__cluster_options = dict(cluster_user='ClusterUser', compute_instance_type='ComputeInstanceType',
-                                      master_instance_type='MasterInstanceType', initial_queue_size='InitialQueueSize',
-                                      max_queue_size='MaxQueueSize', maintain_initial_size='MaintainInitialSize',
-                                      scheduler='Scheduler', cluster_type='ClusterType', ephemeral_dir='EphemeralDir',
-                                      spot_price='SpotPrice', custom_ami='CustomAMI', pre_install='PreInstallScript',
-                                      post_install='PostInstallScript', proxy_server='ProxyServer',
-                                      placement='Placement', placement_group='PlacementGroup',
-                                      encrypted_ephemeral='EncryptedEphemeral',pre_install_args='PreInstallArgs',
-                                      post_install_args='PostInstallArgs', s3_read_resource='S3ReadResource',
-                                      s3_read_write_resource='S3ReadWriteResource')
+        self.__cluster_options = dict(cluster_user=('ClusterUser', None), compute_instance_type=('ComputeInstanceType',None),
+                                      master_instance_type=('MasterInstanceType', None), initial_queue_size=('InitialQueueSize',None),
+                                      max_queue_size=('MaxQueueSize',None), maintain_initial_size=('MaintainInitialSize',None),
+                                      scheduler=('Scheduler',None), cluster_type=('ClusterType',None), ephemeral_dir=('EphemeralDir',None),
+                                      spot_price=('SpotPrice',None), custom_ami=('CustomAMI','EC2Ami'), pre_install=('PreInstallScript','URL'),
+                                      post_install=('PostInstallScript','URL'), proxy_server=('ProxyServer',None),
+                                      placement=('Placement',None), placement_group=('PlacementGroup','EC2PlacementGroup'),
+                                      encrypted_ephemeral=('EncryptedEphemeral',None),pre_install_args=('PreInstallArgs',None),
+                                      post_install_args=('PostInstallArgs',None), s3_read_resource=('S3ReadResource',None),
+                                      s3_read_write_resource=('S3ReadWriteResource',None))
 
         # Loop over all the cluster options and add define to parameters, raise Exception if defined but null
         for key in self.__cluster_options:
             try:
                 __temp__ = __config.get(self.__cluster_section, key)
                 if not __temp__:
                     raise Exception
-                self.parameters.append((self.__cluster_options.get(key),__temp__))
+                if self.__sanity_check and self.__cluster_options.get(key)[1] is not None:
+                    config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                                self.__cluster_options.get(key)[1],__temp__)
+                self.parameters.append((self.__cluster_options.get(key)[0],__temp__))
             except ConfigParser.NoOptionError:
                 pass
 
@@ -181,8 +191,9 @@ def __init__(self, args):
             pass
 
         # Dictionary list of all EBS options
-        self.__ebs_options = dict(ebs_snapshot_id='EBSSnapshotId', volume_type='VolumeType', volume_size='VolumeSize',
-                                  volume_iops='VolumeIOPS', encrypted='EBSEncryption')
+        self.__ebs_options = dict(ebs_snapshot_id=('EBSSnapshotId','EC2Snapshot'), volume_type=('VolumeType',None),
+                                  volume_size=('VolumeSize',None),
+                                  volume_iops=('VolumeIOPS',None), encrypted=('EBSEncryption',None))
 
         try:
             if self.__ebs_section:
@@ -191,7 +202,10 @@ def __init__(self, args):
                         __temp__ = __config.get(self.__ebs_section, key)
                         if not __temp__:
                             raise Exception
-                        self.parameters.append((self.__ebs_options.get(key),__temp__))
+                        if self.__sanity_check and self.__ebs_options.get(key)[1] is not None:
+                            config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                                self.__ebs_options.get(key)[1],__temp__)
+                        self.parameters.append((self.__ebs_options.get(key)[0],__temp__))
                     except ConfigParser.NoOptionError:
                         pass
         except AttributeError:
@@ -207,8 +221,10 @@ def __init__(self, args):
             pass
 
         # Dictionary list of all scaling options
-        self.__scaling_options = dict(scaling_threshold='ScalingThreshold', scaling_period='ScalingPeriod',
-                                      scaling_evaluation_periods='ScalingEvaluationPeriods')
+        self.__scaling_options = dict(scaling_threshold=('ScalingThreshold',None), scaling_period=('ScalingPeriod',None),
+                                      scaling_evaluation_periods=('ScalingEvaluationPeriods',None),
+                                      scaling_adjustment=('ScalingAdjustment',None),scaling_adjustment2=('ScalingAdjustment2',None),
+                                      scaling_cooldonw=('ScalingCooldown',None),scaling_threshold2=('ScalingThreshold2',None))
 
         try:
             if self.__scaling_section:
@@ -217,7 +233,10 @@ def __init__(self, args):
                         __temp__ = __config.get(self.__scaling_section, key)
                         if not __temp__:
                             raise Exception
-                        self.parameters.append((self.__scaling_options.get(key),__temp__))
+                        if self.__sanity_check and self.__scaling_options.get(key)[1] is not None:
+                            config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key,
+                                                self.__scaling_options.get(key)[1],__temp__)
+                        self.parameters.append((self.__scaling_options.get(key)[0],__temp__))
                     except ConfigParser.NoOptionError:
                         pass
         except AttributeError:

cli/cfncluster/cli.py

@@ -39,10 +39,15 @@ def update(args):
 
 def main():
     # set up logging to file
+
     if platform.system() is 'Windows':
+        if not os.path.exists(os.path.expanduser('~\.cfncluster')):
+            os.makedirs(os.path.expanduser('~\.cfncluster'))
         logfile = os.path.expanduser('~\.cfncluster\cfncluster-cli.log')
     else:
-        logfile = '/tmp/cfncluster-cli.log'
+        if not os.path.exists(os.path.expanduser('~/.cfncluster')):
+            os.makedirs(os.path.expanduser('~/.cfncluster'))
+        logfile = os.path.expanduser('~/.cfncluster/cfncluster-cli.log')
     logging.basicConfig(level=logging.DEBUG,
                         format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                         datefmt='%m-%d %H:%M',

cli/cfncluster/config_sanity.py

@@ -0,0 +1,86 @@
+# Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Amazon Software License (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/asl/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'dougalb'
+
+import boto.ec2
+import boto.vpc
+import urllib2
+import boto.exception
+import sys
+
+def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_type,resource_value):
+
+    # Loop over all supported resource checks
+    # EC2 KeyPair
+    if resource_type == 'EC2KeyPair':
+        try:
+            ec2_conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = ec2_conn.get_all_key_pairs(keynames=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)
+    # VPC Id
+    elif resource_type == 'VPC':
+        try:
+            vpc_conn = boto.vpc.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = vpc_conn.get_all_vpcs(vpc_ids=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)
+    # VPC Subnet Id
+    elif resource_type == 'VPCSubnet':
+        try:
+            vpc_conn = boto.vpc.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = vpc_conn.get_all_subnets(subnet_ids=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)
+    # EC2 AMI Id
+    elif resource_type == 'EC2Ami':
+        try:
+            ec2_conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = ec2_conn.get_all_images(image_ids=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)
+    # EC2 Placement Group
+    elif resource_type == 'EC2PlacementGroup':
+        try:
+            ec2_conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = ec2_conn.get_all_placement_groups(groupnames=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)
+    # URL
+    elif resource_type == 'URL':
+        try:
+            urllib2.urlopen(resource_value)
+        except urllib2.HTTPError, e:
+            print(e.code)
+            sys.exit(1)
+        except urllib2.URLError, e:
+            print(e.args)
+            sys.exit(1)
+    # EC2 EBS Snapshot Id
+    elif resource_type == 'EC2Snapshot':
+        try:
+            ec2_conn = boto.ec2.connect_to_region(region,aws_access_key_id=aws_access_key_id,
+                                                 aws_secret_access_key=aws_secret_access_key)
+            test = ec2_conn.get_all_snapshots(snapshot_ids=resource_value)
+        except boto.exception.BotoServerError as e:
+            print('Config sanity error: %s' % e.message)
+            sys.exit(1)

cli/cfncluster/examples/config

@@ -4,6 +4,8 @@
 cluster_template = default
 # Check for updates
 update_check = true
+# Check config sanity. (Attempts to validate resources defined in parameters, actually exist)
+sanity_check = true
 
 [aws]
 # This is the AWS credentials section (required).
@@ -16,9 +18,6 @@ update_check = true
 # Uncomment to specify a different Amazon AWS region  (OPTIONAL)
 #aws_region_name = us-west-2
 
-[keypair mykey]
-key_location = /path/to/key.pem
-
 ## cfncluster templates
 [cluster default]
 # Name of an existing EC2 KeyPair to enable SSH access to the instances.
@@ -87,6 +86,9 @@ key_name = mykey
 # Encrypted ephemeral drives. In-memory keys, non-recoverable.
 # (defaults to false in default template)
 #encrypted_ephemeral = false
+# OS type used in the cluster
+# (defaults to centos6 in the default template)
+#base_os = centos6
 # Settings section relating to VPC to be used
 vpc_settings = public
 # Settings section relating to EBS volume
@@ -143,9 +145,21 @@ master_subnet_id = subnet-
 # Threshold for triggering CloudWatch ScaleUp action
 # (defaults to 4 for default template)
 #scaling_threshold = 4
+# Number of instances to add when called CloudWatch ScaleUp action
+# (defaults to 2 for default template)
+#scaling_adjustment = 2
+# Threshold for triggering CloudWatch ScaleUp action
+# (defaults to 4 for default template)
+#scaling_threshold2 = 200
+# Number of instances to add when called CloudWatch ScaleUp2 action
+# (defaults to 20 for default template)
+#scaling_adjustment = 20
 # Period to measure ScalingThreshold
 # (defaults to 60 for default template)
 #scaling_period = 60
 # Period to measure ScalingThreshold
 # (defaults to 2 for default template)
-#scaling_evaluation_periods = 2
+#scaling_evaluation_periods = 2
+# Amount of time in seconds to wait before attempting further scaling actions
+# (defaults to 120 for the default template
+#scaling_cooldown = 120

cli/setup.py

@@ -20,8 +20,8 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 console_scripts = ['cfncluster = cfncluster.cli:main']
-version = "0.0.13"
-requires = ['boto>=2.32.1', 'botocore'] 
+version = "0.0.99"
+requires = ['boto>=2.33', 'botocore'] 
 
 if sys.version_info[:2] == (2, 6):
     # For python2.6 we have to require argparse since it