Merge pull request #14 from cloudify-cosmo/RD-1478

try to minimize some lookups

Author: EarthmanT

.circleci/devscript.py

@@ -0,0 +1,203 @@
+import os
+import urllib3
+from time import sleep
+from datetime import datetime
+
+from vcd_plugin_sdk.resources.disk import VCloudDisk, VCloudMedia, VCloudISO
+from vcd_plugin_sdk.resources.vapp import VCloudvApp, VCloudVM
+from vcd_plugin_sdk.resources.network import VCloudNetwork, VCloudGateway
+# from vcd_plugin_sdk.resources.storage_profile import VCloudStorageProfile
+# urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+NOW = datetime.now().strftime("%Y%m%d%H%M")
+NETWORK_NAME = 'test-routed-network-{}'.format(NOW)
+GATEWAY_NAME = os.environ['VCLOUD_GATEWAY']
+VCD_NAME = os.environ['VCLOUD_ORG']
+
+
+def get_gateway():
+    # static_route = dict(network='192.169.3.0/24', next_hop='192.168.1.1')
+    # static_route_create_result = gateway.add_static_route(**static_route)
+    # gateway.delete_static_route(**static_route)
+    #
+    #
+    # # Test Firewall Rule
+    # test_rule_name = 'test-rule-{}'.format(rnow.strftime("%Y%m%d%H%M"))
+    # firewall_rule_info = gateway.create_firewall_rule(test_rule_name,
+    # source_values=['VLAN-102' + ':gatewayinterface',
+    # routed_network.network.name + ':network', '192.169.1.0:ip'],
+    # destination_values=['VLAN-102' + ':gatewayinterface',
+    # routed_network.network.name + ':network', '192.169.1.0:ip'], services=[
+    # {'tcp': {'any': 'any'}}])
+    # gateway.delete_firewall_rule(firewall_rule_info['Name'],
+    # firewall_rule_info['Id'])
+    #
+    # # Test NAT rule
+    # nat_rule = dict(action='dnat', original_address='10.10.4.2',
+    # translated_address='11.11.4.2', description='nat rule test value')
+    # new_nat_rule = gateway.create_nat_rule(nat_rule)
+    # gateway.delete_nat_rule(new_nat_rule['ID'])
+    #
+    # # Test DHCP Pool
+    # new_dhcp_rule = gateway.add_dhcp_pool(dict(
+    # ip_range='192.169.2.2-192.169.2.100'))
+    # gateway.delete_dhcp_pool(dict(ip_range='192.169.2.2-192.169.2.100'))
+    return VCloudGateway(NETWORK_NAME, vdc_name=VCD_NAME)
+
+
+def get_routed_network():
+    create_routed = {
+        'network_cidr': '192.169.169.1/24',
+        'gateway_name': GATEWAY_NAME,
+        'description': 'test routed network',
+        'primary_dns_ip': '8.8.8.4',
+        'secondary_dns_ip': '8.8.8.8',
+        'ip_range_start': '192.169.169.2',
+        'ip_range_end': '192.169.169.254'
+    }
+    network = VCloudNetwork(
+        NETWORK_NAME,
+        'routed_vdc_network',
+        vdc_name=VCD_NAME,
+        vapp_name='test',
+        kwargs=create_routed
+    )
+    network.create()
+    return network
+
+
+def get_isolated_network():
+    create_isolated = {
+        'network_cidr': '192.169.3.1/24',
+        'description': 'test isolated network',
+        'primary_dns_ip': '8.8.8.4',
+        'secondary_dns_ip': '8.8.8.8',
+        'default_lease_time': 300,
+        'max_lease_time': 900,
+    }
+    isolated_network = VCloudNetwork(
+        'test-iso-network-{}'.format(NOW),
+        'isolated_vdc_network',
+        vdc_name=VCD_NAME,
+        vapp_name='test',
+        kwargs=create_isolated
+    )
+    isolated_network.create()
+    return isolated_network
+
+
+def get_vapp(network_name):
+
+    create_vapp = {
+        'description': 'test description',
+        'network': network_name,
+        'fence_mode': 'natRouted',
+        'accept_all_eulas': True
+    }
+    vapp_name = 'test-vapp-{}'.format(NOW)
+    vapp = VCloudvApp(
+        vapp_name,
+        vdc_name=VCD_NAME,
+        kwargs=create_vapp
+    )
+    catalog_name = 'testcatalogue'
+    template = vapp.get_template('testcatalogue', 'Centos7-GenericCloud')
+
+    instantiate_vapp = {
+        'name': 'test-vm-{}'.format(NOW),
+        'catalog': catalog_name,
+        'template': template.get('name'),
+        'description': 'test description',
+        'network': routed_network.name,
+        'fence_mode': 'bridged',
+        'ip_allocation_mode': 'manual',
+        'deploy': False,
+        'power_on': False,
+        'accept_all_eulas': True,
+        'password': 'test_password',
+        'vm_name': 'test-vm-{}'.format(NOW),
+        'hostname': 'test-vm-{}'.format(NOW),
+        'ip_address': '192.169.169.2',
+    }
+    vm = VCloudVM(
+        'test-vm-{}'.format(NOW),
+        vdc_name=VCD_NAME,
+        vapp_name=vapp_name,
+        kwargs={},
+        vapp_kwargs=instantiate_vapp)
+    vm.instantiate_vapp()
+    sleep(60)
+    vapp.deploy()
+    # To add network: add_network_result = vapp.add_network(
+    # orgvdc_network_name=routed_network2.name)
+    # To remove network: delete_network_result = vapp.remove_network(
+    # routed_network2.name)
+    return vapp
+
+
+def get_disk():
+    create_disk = {
+        'size': 2097152,
+        'description': 'test disk'
+    }
+    disk_now = datetime.now()
+    disk_name = 'test-disk-{}'.format(disk_now.strftime("%Y%m%d%H%M"))
+    disk = VCloudDisk(
+        disk_name,
+        vdc_name='vCloud97',
+        kwargs=create_disk
+    )
+    disk.create()
+    # To attach: attach_disk_result = vm.attach_disk_to_vm(disk.href)
+    # To detach: vm.detach_disk_from_vm(disk.href)
+    return disk
+
+
+def get_media(catalog_name):
+    # To get catalog_name, check get_vapp.
+    create_iso = {
+        'vol_ident': 'cidata',
+        'sys_ident': '',
+        'files': {
+            'ISO/FOLDER/content.json': "test content"
+        }
+    }
+    iso = VCloudISO(create_iso)
+    media_create = {
+        'catalog_name': catalog_name,
+        'file_name': iso.file,
+    }
+    media = VCloudMedia(
+        os.path.basename(iso.file),
+        vdc_name='vCloud97',
+        kwargs=media_create,
+    )
+    media.upload()
+    # To delete execute media.delete() iso.delete()
+    # To attach: attach_media_result = vm.attach_media(media.href)
+    # To detaych: vm.eject_media(media.name)
+    return media
+
+
+if __name__ == '__main__':
+    print('**step 1**')
+    gateway = get_gateway()
+    print('**step 2**')
+    routed_network = get_routed_network()
+    print('**step 3**')
+    isolated_network = get_isolated_network()
+    print('**step 4**')
+    vapp = get_vapp(routed_network.name)
+
+    sleep(60)
+
+    print('**step 5**')
+    vapp.undeploy()
+    sleep(20)
+    print('**step 6**')
+    vapp.delete()
+    print('**step 7**')
+    isolated_network.delete()
+    print('**step 8**')
+    routed_network.delete()

.circleci/merge_docs.py

@@ -1,4 +1,8 @@
 from ecosystem_cicd_tools.github_stuff import merge_documentation_pulls
 
 if __name__ == '__main__':
+<<<<<<< HEAD
     merge_documentation_pulls(branch='main')
+=======
+    merge_documentation_pulls()
+>>>>>>> af5df53 (try to minimize some lookups)

.circleci/py3fixers

@@ -11,8 +11,6 @@
 -f libfuturize.fixes.fix_execfile
 -f libpasteurize.fixes.fix_newstyle
 -f lib2to3.fixes.fix_filter
-# fix_dict is not idempotent
-# -f lib2to3.fixes.fix_dict
 -f lib2to3.fixes.fix_map
 -f lib2to3.fixes.fix_zip
 -f lib2to3.fixes.fix_xrange
@@ -22,4 +20,4 @@
 -f lib2to3.fixes.fix_metaclass
 -f libfuturize.fixes.fix_unicode_keep_u
 -f libfuturize.fixes.fix_print_with_import
--f libfuturize.fixes.fix_execfile
+-f libfuturize.fixes.fix_execfile

CHANGELOG.txt

@@ -1,2 +1,4 @@
 2.0.0:
-  - New version.
+  - New version.
+2.0.1:
+  - Attempt to decrease amount of superfluous API requests, however these calls are almost all inside of the client library.

plugin.yaml

@@ -3,7 +3,7 @@ plugins:
   vcd:
     executor: central_deployment_agent
     package_name: cloudify-vcloud-plugin
-    package_version: '2.0.0'
+    package_version: '2.0.1'
 
 data_types:
 

vcd_plugin_sdk/resources/base.py

@@ -52,8 +52,10 @@ def connection(self):
 
     @property
     def vdc(self):
-        if self._vdc:
-            self._vdc.reload()
+        # TODO: Create a method for reloading when necessary. For example,
+        # TODO: as part of a try/except block.
+        # if self._vdc:
+        #     self._vdc.reload()
         return self._vdc
 
     def task_successful(self, task):
@@ -78,10 +80,10 @@ def vapp(self):
                 'vcd_plugin_sdk.resources.base.VCloudResource.vapp, '
                 'but no vapp_name was provided.')
         if not self._vapp:
-            self._vapp = self.get_vapp(self._vapp_name)
+            self._vapp = self.get_vapp()
         return self._vapp
 
-    def get_vapp(self, vapp_name):
+    def get_vapp(self, vapp_name=None):
         vapp_name = vapp_name or self._vapp_name
         try:
             vapp_resource = self.vdc.get_vapp(vapp_name)

vcd_plugin_sdk/resources/network.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from time import sleep
+
 from pyvcloud.vcd.gateway import Gateway
 from pyvcloud.vcd.nat_rule import NatRule
 from pyvcloud.vcd.dhcp_pool import DhcpPool
@@ -63,13 +65,15 @@ def name(self):
     def network(self):
         if not self._network:
             for i in range(0, 10):
+                # This is necessary because of Vcloud API is async.
                 try:
                     self._network = self.get_network()
                 except (ValueError, EntityNotFoundException) as e:
                     if i == 9:
                         raise VCloudSDKException(
                             'Network {name} has not been initialized. '
                             'Error: {e}'.format(name=self.name, e=str(e)))
+                    sleep(5)
         return self._network
 
     @property
@@ -199,6 +203,7 @@ def __init__(self,
         self._gateway_name = gateway_name
         self.kwargs = kwargs
         self._gateway = None
+        self._gateway_static_routes = None
 
         super().__init__(connection, vdc_name, tasks=tasks)
 
@@ -208,9 +213,7 @@ def name(self):
 
     @property
     def gateway(self):
-        if self._gateway:
-            self._gateway.reload()
-        else:
+        if not self._gateway:
             self._gateway = self.get_gateway()
         return self._gateway
 
@@ -229,15 +232,13 @@ def firewall_objects(self):
 
     @property
     def default_gateway(self):
-        static_route = self.gateway.get_static_routes()
-        return static_route.defaultRoute.gatewayAddress.text
+        return self.gateway_static_routes.defaultRoute.gatewayAddress.text
 
     @property
     def static_routes(self):
         static_routes = {}
-        static_route = self.gateway.get_static_routes()
-        if hasattr(static_route.staticRoutes, 'route'):
-            for route in static_route.staticRoutes.route:
+        if hasattr(self.gateway_static_routes.staticRoutes, 'route'):
+            for route in self.gateway_static_routes.staticRoutes.route:
                 static_routes[route.network] = {
                     #  'mtu': route.mtu,
                     'description': route.description,
@@ -265,12 +266,21 @@ def exposed_data(self):
             'gateway_address': self.default_gateway,
             # 'dhcp_pools': self.dhcp_pools,
             # 'nat_rules': self.nat_rules,
-            # 'static_routes': self.static_routes,
+            'static_routes': self.static_routes,
             # 'firewall_rules': self.firewall_rules,
             # 'firewall_objects': self.firewall_objects
         }
         return data
 
+    @property
+    def gateway_static_routes(self):
+        if not self._gateway_static_routes:
+            self.update_gateway_static_routes()
+        return self._gateway_static_routes
+
+    def update_gateway_static_routes(self):
+        self._gateway_static_routes = self.gateway.get_static_routes()
+
     def get_gateway(self, gateway_name=None):
         gateway_name = gateway_name or self.name
         gateway_resource = self.vdc.get_gateway(gateway_name)
@@ -408,6 +418,7 @@ def get_static_routes(self):
         return static_routes
 
     def get_static_route_from_network(self, network):
+        self.update_gateway_static_routes()
         for route in self.get_static_routes():
             if route.resource_id == network:
                 return route

vcd_plugin_sdk/resources/tests/test_resources.py

@@ -318,7 +318,7 @@ def test_vcloud_gateway_dhcp_pool(*_, **__):
                           mock.MagicMock)
 
     vcloud_gateway.get_dhcp_pool_from_ip_range(pool_definition['ip_range'])
-    assert vcloud_gateway.client.get_resource.call_count == 4
+    assert vcloud_gateway.client.get_resource.call_count == 2
 
 
 @mock.patch('vcd_plugin_sdk.connection.Org', autospec=True)
@@ -396,9 +396,9 @@ def test_vcloud_vapp(*_, **__):
         with mock.patch('lxml.etree.cleanup_namespaces'):
             vcloud_vapp.set_lease(1, 1)
             assert vcloud_vapp.client.put_resource.call_count == 1
-            assert vcloud_vapp.client.get_resource.call_count == 9
+            assert vcloud_vapp.client.get_resource.call_count == 6
             vcloud_vapp.get_lease()
-            assert vcloud_vapp.client.get_resource.call_count == 10
+            assert vcloud_vapp.client.get_resource.call_count == 7
     vcloud_vapp.remove_network('bar')
     assert vcloud_vapp.vapp.disconnect_org_vdc_network.call_count == 1
 

vcd_plugin_sdk/resources/vapp.py

@@ -92,7 +92,7 @@ def get_catalog_items(self):
                     c.CatalogItems.CatalogItem.items()
         return items
 
-    def get_vapp(self, vapp_name):
+    def get_vapp(self, vapp_name=None):
         vapp_resource = self.vdc.get_vapp(vapp_name)
         return VApp(self.client, resource=vapp_resource)