Cy 837 update sdk library versions (#171)

* CY-837 Update Cloudify AWSSDK Plugin SDK Library Versions

* CY-837 Update rds cloudformation exmaple

* Route53 example updated

* CY-865: Remove references to old aws plugin from blueprint examples

* CY-865: Update elb blueprint examples

* CY-865: Update natgateway blueprint example

* CY-865: Update lambda example

* CY-865: Update kms example

* CY-865: Update ecs example

* CY-865: fix issue with autoscaling group

* CY-865: Update ecs example

* CY-865: Update ec2 examples

* CY-865: Update ec2 examples

* CY-865: Update blueprint example and use the latest version for cloudify

* CY-865: Update plugin to add support for backward compatibility with old aws plugin

* Temporary change to rerun integration tests on CI (all tests)

* CY-892: Add cloudify manager integration tests

* CY-837: Update awssdk test

* CY-837: Update awssdk plugin to verion 2.8.0

Author: mabuaisha

.circleci/config.yml

@@ -107,10 +107,10 @@ jobs:
           command: pip install -e .
       - run:
           name: install test requirements
-          command: pip install awscli nose testtools https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.zip https://github.com/cloudify-incubator/cloudify-utilities-plugin/archive/1.7.1.zip https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/2.0.zip
+          command: pip install awscli nose testtools https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.zip https://github.com/cloudify-incubator/cloudify-utilities-plugin/archive/1.7.1.zip https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/2.1.2.zip
       - run:
           name: execute test
-          command: nosetests -s integration_tests/test_awssdk.py
+          command: nosetests -s integration_tesing/test_awssdk.py
 
 workflows:
   version: 2
@@ -125,7 +125,6 @@ workflows:
           context: ecosystem
           requires:
             - wagon
-            - unittests
           filters:
             branches:
               only: /([0-9\.]*\-build|master|dev)/

cloudify_awssdk/autoscaling/resources/autoscaling_group.py

@@ -16,7 +16,11 @@
     Autoscaling.Group
     ~~~~~~~~~~~~~~
     AWS Autoscaling Group interface
+
 """
+# Standard imports
+from re import sub
+
 # Cloudify
 from cloudify.exceptions import OperationRetry
 from cloudify_awssdk.common import decorators, utils
@@ -155,7 +159,7 @@ def create(ctx, iface, resource_config, **_):
 
     subnet_list_string = params.get(SUBNET_LIST)
     subnet_list = \
-        subnet_list_string.split(', ') if \
+        sub("[^\w]", " ", subnet_list_string).split() if \
         subnet_list_string else []
 
     subnet_list = \
@@ -168,7 +172,10 @@ def create(ctx, iface, resource_config, **_):
             ctx.instance,
             SUBNET_TYPE_DEPRECATED,
             subnet_list)
+
     if subnet_list:
+        # Remove any duplicate items from subnet list
+        subnet_list = list(set(subnet_list))
         params[SUBNET_LIST] = ', '.join(subnet_list)
 
     # Actually create the resource

cloudify_awssdk/autoscaling/resources/launch_configuration.py

@@ -20,13 +20,15 @@
 # Cloudify
 from cloudify_awssdk.common import decorators, utils
 from cloudify_awssdk.autoscaling import AutoscalingBase
+from cloudify.exceptions import NonRecoverableError
 # Boto
 from botocore.exceptions import ClientError
 
 RESOURCE_TYPE = 'Autoscaling Launch Configuration'
 LCS = 'LaunchConfigurations'
 RESOURCE_NAMES = 'LaunchConfigurationNames'
 RESOURCE_NAME = 'LaunchConfigurationName'
+IAM_INSTANCE_PROFILE = 'IamInstanceProfile'
 LC_ARN = 'LaunchConfigurationARN'
 IMAGEID = 'ImageId'
 INSTANCEID = 'InstanceId'
@@ -106,6 +108,17 @@ def create(ctx, iface, resource_config, **_):
     params[RESOURCE_NAME] = resource_id
     utils.update_resource_id(ctx.instance, resource_id)
 
+    # Check if the "IamInstanceProfile" is passed or not and then update it
+    iam_instance_profile = params.get(IAM_INSTANCE_PROFILE)
+    if iam_instance_profile:
+        if isinstance(iam_instance_profile, basestring):
+            iam_instance_profile = iam_instance_profile.strip()
+            params[IAM_INSTANCE_PROFILE] = str(iam_instance_profile)
+        else:
+            raise NonRecoverableError(
+                'Invalid {0} data type for {1}'
+                ''.format(type(iam_instance_profile), IAM_INSTANCE_PROFILE))
+
     # Add Security Groups
     secgroups_list = params.get(SECGROUPS, [])
     params[SECGROUPS] = \

cloudify_awssdk/cloudformation/tests/test_stack.py

@@ -12,6 +12,7 @@
 #    * See the License for the specific language governing permissions and
 #    * limitations under the License.
 import unittest
+import copy
 from mock import patch, MagicMock
 
 from cloudify.state import current_ctx
@@ -73,10 +74,11 @@ def test_prepare(self):
                             type_class=stack)
 
     def test_create(self):
-        _ctx = self.get_mock_ctx('test_create',
-                                 test_properties=NODE_PROPERTIES,
-                                 test_runtime_properties=RUNTIME_PROPERTIES,
-                                 type_hierarchy=STACK_TH)
+        _ctx = self.get_mock_ctx(
+            'test_create', test_properties=NODE_PROPERTIES,
+            test_runtime_properties=RUNTIME_PROPERTIES,
+            type_hierarchy=STACK_TH,
+            ctx_operation_name='cloudify.interfaces.lifecycle.configure')
 
         current_ctx.set(_ctx)
         self.fake_client.describe_stacks = MagicMock(return_value={
@@ -106,8 +108,13 @@ def test_create(self):
             except AssertionError as e:
                 raise e
 
+        updated_runtime_prop = copy.deepcopy(RUNTIMEPROP_AFTER_CREATE)
+        updated_runtime_prop['create_response'] = {
+            'StackName': 'Stack',
+            'StackStatus': 'CREATE_COMPLETE'
+        }
         self.assertEqual(_ctx.instance.runtime_properties,
-                         RUNTIMEPROP_AFTER_CREATE)
+                         updated_runtime_prop)
 
     def test_delete(self):
         _ctx = \

cloudify_awssdk/common/decorators.py

@@ -217,6 +217,7 @@ def wrapper_outer(function):
         def wrapper_inner(**kwargs):
             '''Inner, worker function'''
             ctx = kwargs['ctx']
+            _, _, _, operation_name = ctx.operation.name.split('.')
             resource_type = kwargs.get('resource_type', 'AWS Resource')
             iface = kwargs['iface']
             # Run the operation if this is the first pass
@@ -251,6 +252,12 @@ def wrapper_inner(**kwargs):
                 raise OperationRetry(
                     '%s ID# "%s" is still in a pending state.'
                     % (resource_type, iface.resource_id))
+
+            elif status_good and status in status_good:
+                if operation_name in ['create', 'configure']:
+                    ctx.instance.runtime_properties['create_response'] = \
+                        utils.JsonCleanuper(iface.properties).to_dict()
+
             elif not status and fail_on_missing:
                 raise NonRecoverableError(
                     '%s ID# "%s" no longer exists but "fail_on_missing" set'

cloudify_awssdk/common/tests/test_decorators.py

@@ -77,7 +77,9 @@ def test_delete(*agrs, **kwargs):
 
     def test_wait_for_status(self):
 
-        _ctx = self._gen_decorators_context('test_wait_for_status')
+        _ctx = self._gen_decorators_context(
+            'test_wait_for_status',
+            op_name='cloudify.interfaces.lifecycle.create')
 
         @decorators.wait_for_status(status_good=['ok'],
                                     status_pending=['pending'])
@@ -87,37 +89,43 @@ def test_ok(*agrs, **kwargs):
         # ok
         mock_interface = MagicMock()
         mock_interface.status = 'ok'
+        mock_interface.properties = {'status': 'ok'}
 
         test_ok(ctx=_ctx, iface=mock_interface)
 
         self.assertEqual(_ctx.instance.runtime_properties, {
-            'resource_config': {}
+            'resource_config': {},
+            'create_response': {'status': 'ok'},
         })
 
         # pending
         mock_interface = MagicMock()
         mock_interface.status = 'pending'
+        mock_interface.properties = {'status': 'pending'}
 
         with self.assertRaises(OperationRetry):
             test_ok(ctx=_ctx, iface=mock_interface)
 
         # unknow
         mock_interface = MagicMock()
-        mock_interface.status = 'unknow'
+        mock_interface.status = 'unknown'
+        mock_interface.properties = {'status': 'unknown'}
 
         with self.assertRaises(NonRecoverableError):
             test_ok(ctx=_ctx, iface=mock_interface)
 
         # empty status
         mock_interface = MagicMock()
         mock_interface.status = None
+        mock_interface.properties = {'status': None}
 
         with self.assertRaises(NonRecoverableError):
             test_ok(ctx=_ctx, iface=mock_interface)
 
         # empty status but ignore
         mock_interface = MagicMock()
         mock_interface.status = None
+        mock_interface.properties = {'status': None}
 
         @decorators.wait_for_status(status_pending=['pending'],
                                     fail_on_missing=False)

cloudify_awssdk/dynamodb/tests/test_table.py

@@ -14,6 +14,7 @@
 
 from mock import patch, MagicMock
 import unittest
+import copy
 from cloudify.state import current_ctx
 from cloudify_awssdk.common.tests.test_base import TestBase, CLIENT_CONFIG
 from cloudify_awssdk.common.tests.test_base import DELETE_RESPONSE
@@ -67,7 +68,8 @@ def test_create(self):
             'test_create',
             test_properties=NODE_PROPERTIES,
             test_runtime_properties=DEFAULT_RUNTIME_PROPERTIES,
-            type_hierarchy=TABLE_TH
+            type_hierarchy=TABLE_TH,
+            ctx_operation_name='cloudify.interfaces.lifecycle.create',
         )
 
         current_ctx.set(_ctx)
@@ -123,10 +125,11 @@ def test_create(self):
             TableName='aws_table_name'
         )
 
-        self.assertEqual(
-            _ctx.instance.runtime_properties,
-            RUNTIME_PROPERTIES_AFTER_CREATE
-        )
+        updated_runtime_prop = copy.deepcopy(RUNTIME_PROPERTIES_AFTER_CREATE)
+        updated_runtime_prop['create_response'] = {'TableStatus': 'ACTIVE'}
+
+        self.assertEqual(_ctx.instance.runtime_properties,
+                         updated_runtime_prop)
 
     def test_delete(self):
         _ctx = self.get_mock_ctx(

cloudify_awssdk/ec2/resources/elasticip.py

@@ -238,7 +238,8 @@ def detach(ctx, iface, resource_config, **_):
     if not elasticip_id:
         elasticip_id = iface.resource_id
         params[ELASTICIP_ID] = elasticip_id
-    elif not association_id:
+
+    if not association_id:
         return
 
     if association_id and elasticip_id:

cloudify_awssdk/ec2/resources/subnet.py

@@ -31,6 +31,7 @@
 SUBNET_ID = 'SubnetId'
 SUBNET_IDS = 'SubnetIds'
 CIDR_BLOCK = 'CidrBlock'
+IPV6_CIDR_BLOCK = 'Ipv6CidrBlock'
 VPC_ID = 'VpcId'
 VPC_TYPE = 'cloudify.nodes.aws.ec2.Vpc'
 VPC_TYPE_DEPRECATED = 'cloudify.aws.nodes.Vpc'
@@ -81,14 +82,14 @@ def delete(self, params=None):
         self.logger.debug('Response: %s' % res)
         return res
 
-    def modify_instance_attribute(self, params=None):
+    def modify_subnet_attribute(self, params=None):
         '''
             Modifies an existing AWS EC2 Subnet Attribute.
         '''
         self.logger.debug(
             'Modifying {0} attribute with parameters: {1}'.format(
                 self.type_name, params))
-        res = self.client.modify_instance_attribute(**params)
+        res = self.client.modify_subnet_attribute(**params)
         self.logger.debug('Response: %s' % res)
         return res
 
@@ -110,6 +111,7 @@ def create(ctx, iface, resource_config, **_):
 
     vpc_id = params.get(VPC_ID)
     cidr_block = params.get(CIDR_BLOCK)
+    ipv6_cidr_block = params.get(IPV6_CIDR_BLOCK)
 
     # If either of these values is missing,
     # they must be filled from a connected VPC.
@@ -134,6 +136,12 @@ def create(ctx, iface, resource_config, **_):
             targ.instance.runtime_properties.get(
                 'resource_config', {}).get(CIDR_BLOCK)
 
+    # If ipv6 cidr block is provided by user, then we need to make sure that
+    # The subnet size must use a /64 prefix length
+    if ipv6_cidr_block:
+        ipv6_cidr_block = ipv6_cidr_block[:-2] + '64'
+        params[IPV6_CIDR_BLOCK] = ipv6_cidr_block
+
     # Actually create the resource
     create_response = iface.create(params)[SUBNET]
     ctx.instance.runtime_properties['create_response'] = \

cloudify_awssdk/ec2/resources/vpc.py

@@ -109,6 +109,7 @@ def create(ctx, iface, resource_config, **_):
     create_response = iface.create(params)[VPC]
     ctx.instance.runtime_properties['create_response'] = \
         utils.JsonCleanuper(create_response).to_dict()
+
     vpc_id = create_response.get(VPC_ID, '')
     iface.update_resource_id(vpc_id)
     utils.update_resource_id(