CLOUDSTACK-9287 - Add integration test to cover the private gateway related changes

Author: wilderrodrigues

test/integration/smoke/test_privategw_acl.py

@@ -25,6 +25,7 @@
 from marvin.lib.common import *
 from nose.plugins.attrib import attr
 
+import time
 import logging
 
 class Services:
@@ -231,9 +232,9 @@ def test_01_vpc_privategw_acl(self):
         vpc_off.update(self.apiclient, state='Enabled')
 
         vpc = self.createVPC(vpc_off)
-        
+
         self.cleanup = [vpc, vpc_off, self.account]
-        
+
         physical_networks = get_physical_networks(self.apiclient, self.zone.id)
         if not physical_networks:
             self.fail("No Physical Networks found!")
@@ -317,7 +318,7 @@ def performVPCTests(self, vpc_off, restart_with_cleanup = False):
 
         self.cleanup.insert(0, vm1)
         self.cleanup.insert(0, vm2)
-        
+
         acl1 = self.createACL(vpc_1)
         self.createACLItem(acl1.id, cidr = "0.0.0.0/0")
         privateGw_1 = self.createPvtGw(vpc_1, "10.0.3.100", "10.0.3.101", acl1.id, vlan_1)
@@ -340,19 +341,17 @@ def performVPCTests(self, vpc_off, restart_with_cleanup = False):
         nat_rule_1 = self.create_natrule(vpc_1, vm1, public_ip_1, network_1)
         nat_rule_2 = self.create_natrule(vpc_2, vm2, public_ip_2, network_2)
 
-        self.check_pvt_gw_connectivity(vm1, public_ip_1, vm2.nic[0].ipaddress)
-        self.check_pvt_gw_connectivity(vm2, public_ip_2, vm1.nic[0].ipaddress)
+        self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm1.nic[0].ipaddress])
 
         if restart_with_cleanup:
             self.reboot_vpc_with_cleanup(vpc_1, True)
             self.reboot_vpc_with_cleanup(vpc_2, True)
 
-            self.check_pvt_gw_connectivity(vm1, public_ip_1, vm2.nic[0].ipaddress)
-            self.check_pvt_gw_connectivity(vm2, public_ip_2, vm1.nic[0].ipaddress)
+            self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm1.nic[0].ipaddress])
 
     def performPrivateGWInterfaceTests(self, vpc_off):
         self.logger.debug("Creating VPCs with  offering ID %s" % vpc_off.id)
-        vpc_1 = self.createVPC(vpc_off, cidr = '10.0.1.0/24')
+        vpc_1 = self.createVPC(vpc_off, cidr = '10.0.0.0/16')
 
         self.cleanup = [vpc_1, vpc_off, self.account]
 
@@ -363,85 +362,81 @@ def performPrivateGWInterfaceTests(self, vpc_off):
         vlans = physical_networks[0].vlan.split('-')
         vlan_1 = int(vlans[0])
 
-        network_1 = self.createNetwork(vpc_1, gateway = '10.0.1.1')
+        net_offering_no_lb = "network_offering_no_lb"
+
+        network_1 = self.createNetwork(vpc_1, gateway = '10.0.0.1')
+        network_2 = self.createNetwork(vpc_1, net_offering = net_offering_no_lb, gateway = '10.0.1.1')
+        network_3 = self.createNetwork(vpc_1, net_offering = net_offering_no_lb, gateway = '10.0.2.1')
+        network_4 = self.createNetwork(vpc_1, net_offering = net_offering_no_lb, gateway = '10.0.3.1')
 
         vm1 = self.createVM(network_1)
+        vm2 = self.createVM(network_2)
+        vm3 = self.createVM(network_3)
+        vm4 = self.createVM(network_4)
 
         self.cleanup.insert(0, vm1)
-        
+        self.cleanup.insert(0, vm2)
+        self.cleanup.insert(0, vm3)
+        self.cleanup.insert(0, vm4)
+
         acl1 = self.createACL(vpc_1)
         self.createACLItem(acl1.id, cidr = "0.0.0.0/0")
-        privateGw_1 = self.createPvtGw(vpc_1, "10.0.3.100", "10.0.3.101", acl1.id, vlan_1)
+        privateGw_1 = self.createPvtGw(vpc_1, "10.1.0.100", "10.1.0.101", acl1.id, vlan_1)
         self.replacePvtGwACL(acl1.id, privateGw_1.id)
 
         self.replaceNetworkAcl(acl1.id, network_1)
-
-        staticRoute_1 = self.createStaticRoute(privateGw_1.id, cidr = '10.0.2.0/24')
+        self.replaceNetworkAcl(acl1.id, network_2)
+        self.replaceNetworkAcl(acl1.id, network_3)
+        self.replaceNetworkAcl(acl1.id, network_4)
 
         public_ip_1 = self.acquire_publicip(vpc_1, network_1)
-
         nat_rule_1 = self.create_natrule(vpc_1, vm1, public_ip_1, network_1)
 
         routers = list_routers(self.apiclient,
-                               account=self.account.name,
-                               domainid=self.account.domainid)
-        
+                       account=self.account.name,
+                       domainid=self.account.domainid)
+
         self.assertEqual(isinstance(routers, list), True,
             "Check for list routers response return valid data")
 
         self.assertEqual(len(routers), 2,
             "Check for list routers size returned '%s' instead of 2" % len(routers))
 
-        state_holder = {routers[0].linklocalip : {"state" : None, "mac" : None},
-                        routers[1].linklocalip : {"state" : None, "mac" : None}}
-        state = None
-        mac = None
-        for router in routers:
-            if router.isredundantrouter and router.vpcid:
-                hosts = list_hosts(
-                    self.apiclient,
-                    id=router.hostid)
-                self.assertEqual(
-                    isinstance(hosts, list),
-                    True,
-                    "Check for list hosts response return valid data")
+        self.check_private_gateway_interfaces(routers)
 
-                host = hosts[0]
-                host.user = self.services["configurableData"]["host"]["username"]
-                host.passwd = self.services["configurableData"]["host"]["password"]
-                host.port = self.services["configurableData"]["host"]["port"]
-
-                try:
-                    state = get_process_status(
-                        host.ipaddress,
-                        host.port,
-                        host.user,
-                        host.passwd,
-                        router.linklocalip,
-                        "ip addr | grep eth3 | grep state | awk '{print $9;}'")
-
-                    mac = get_process_status(
-                        host.ipaddress,
-                        host.port,
-                        host.user,
-                        host.passwd,
-                        router.linklocalip,
-                        "ip addr | grep link/ether | awk '{print $2;}' | sed -n 4p")
-                except KeyError:
-                    self.skipTest(
-                        "Provide a marvin config file with host\
-                                credentials to run %s" %
-                        self._testMethodName)
-
-                self.logger.debug("Result from the Router on IP '%s' is -> state: '%s', mac: '%s'" % (router.linklocalip, state, mac))
-                state_holder[router.linklocalip]["state"] = str(state)
-                state_holder[router.linklocalip]["mac"] = str(mac)
+        self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm3.nic[0].ipaddress, vm4.nic[0].ipaddress])
 
-        check_state = state_holder[routers[0].linklocalip]["state"].count(state_holder[routers[1].linklocalip]["state"])
-        check_mac = state_holder[routers[0].linklocalip]["mac"].count(state_holder[routers[1].linklocalip]["mac"])
+        self.reboot_vpc_with_cleanup(vpc_1, True)
 
-        self.assertTrue(check_state == 0, "Routers private gateway interface should not be on the same state!")
-        self.assertTrue(check_mac == 0, "Routers private gateway interface should not have the same mac address!")
+        self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm3.nic[0].ipaddress, vm4.nic[0].ipaddress])
+
+        self.stop_router_by_type(routers, status_to_check = "MASTER")
+        self.check_routers_state(routers)
+
+        self.check_private_gateway_interfaces(routers)
+        self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm3.nic[0].ipaddress, vm4.nic[0].ipaddress])
+
+        self.start_routers(routers)
+        self.check_routers_state(routers)
+        self.check_private_gateway_interfaces(routers)
+        self.check_pvt_gw_connectivity(vm1, public_ip_1, [vm2.nic[0].ipaddress, vm3.nic[0].ipaddress, vm4.nic[0].ipaddress])
+
+    def stop_router_by_type(self, type, routers):
+        self.logger.debug('Stopping %s router' % type)
+        for router in routers:
+            if router.redundantstate == type:
+                self.stop_router(router)
+                break
+
+    def start_routers(self, routers):
+        self.logger.debug('Starting stopped routers')
+        for router in routers:
+            self.logger.debug('Router %s has state %s' % (router.id, router.state))
+            if router.state == "Stopped":
+                self.logger.debug('Starting stopped router %s' % router.id)
+                cmd = startRouter.startRouterCmd()
+                cmd.id = router.id
+                self.apiclient.startRouter(cmd)
 
     def createVPC(self, vpc_offering, cidr = '10.1.1.1/16'):
         try:
@@ -524,10 +519,10 @@ def createACLItem(self, aclId, cidr = "0.0.0.0/0"):
         except Exception, e:
             self.fail('Unable to create ACL Item due to %s ' % e)
 
-    def createNetwork(self, vpc, gateway = '10.1.1.1'):
+    def createNetwork(self, vpc, net_offering = "network_offering", gateway = '10.1.1.1'):
         try:
             self.logger.debug('Create NetworkOffering')
-            net_offerring = self.services["network_offering"]
+            net_offerring = self.services[net_offering]
             net_offerring["name"] = "NET_OFF-%s" % gateway
             nw_off = NetworkOffering.create(
                 self.apiclient,
@@ -584,7 +579,7 @@ def createPvtGw(self, vpc, ip_address, gateway, aclId, vlan):
             self.fail("Failed to create Private Gateway ==> %s" % e)
 
         self.assertIsNotNone(privateGw.id, "Failed to create ACL.")
-        
+
         return privateGw
 
     def replaceNetworkAcl(self, aclId, network):
@@ -643,33 +638,36 @@ def create_natrule(self, vpc, virtual_machine, public_ip, network):
             traffictype='Ingress'
         )
         self.logger.debug('nwacl_nat=%s' % nwacl_nat.__dict__)
-        
+
         return nat_rule
 
-    def check_pvt_gw_connectivity(self, virtual_machine, public_ip, vm_ip):
-        ssh_command = "ping -c 3 %s" % vm_ip
+    def check_pvt_gw_connectivity(self, virtual_machine, public_ip, vms_ips):
+        for vm_ip in vms_ips:
+            ssh_command = "ping -c 3 %s" % vm_ip
 
-        # Should be able to SSH VM
-        result = 'failed'
-        try:
-            self.logger.debug("SSH into VM: %s" % public_ip.ipaddress.ipaddress)
-            
-            ssh = virtual_machine.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress)
+            # Should be able to SSH VM
+            result = 'failed'
+            try:
+                self.logger.debug("SSH into VM: %s" % public_ip.ipaddress.ipaddress)
 
-            self.logger.debug("Ping to VM inside another VPC")
-            result = str(ssh.execute(ssh_command))
+                ssh = virtual_machine.get_ssh_client(ipaddress=public_ip.ipaddress.ipaddress)
 
-            self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3 packets received")))
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (vmObj.get_ip(), e)
-                      )
+                self.logger.debug("Ping to VM inside another Network Tier")
+                result = str(ssh.execute(ssh_command))
 
-        self.assertEqual(
-                         result.count("3 packets received"),
-                         1,
-                         "Ping to outside world from VM should be successful"
-                         )
+                self.logger.debug("SSH result: %s; COUNT is ==> %s" % (result, result.count("3 packets received")))
+            except Exception as e:
+                self.fail("SSH Access failed for %s: %s" % \
+                          (vmObj.get_ip(), e)
+                          )
+
+            self.assertEqual(
+                             result.count("3 packets received"),
+                             1,
+                             "Ping to VM on Network Tier N from VM in Network Tier A should be successful"
+                             )
+
+            time.sleep(5)
 
     def reboot_vpc_with_cleanup(self, vpc, cleanup = True):
         self.logger.debug("Restarting VPC %s with cleanup" % vpc.id)
@@ -680,3 +678,105 @@ def reboot_vpc_with_cleanup(self, vpc, cleanup = True):
         cmd.cleanup = cleanup
         cmd.makeredundant = False
         self.api_client.restartVPC(cmd)
+
+    def check_private_gateway_interfaces(self, routers):
+        state_holder = {routers[0].linklocalip : {"state" : None, "mac" : None},
+                        routers[1].linklocalip : {"state" : None, "mac" : None}}
+        state = None
+        mac = None
+        for router in routers:
+            hosts = list_hosts(self.apiclient, id=router.hostid)
+
+            self.assertEqual(
+                isinstance(hosts, list),
+                True,
+                "Check for list hosts response return valid data")
+
+            host = hosts[0]
+            host.user = self.services["configurableData"]["host"]["username"]
+            host.passwd = self.services["configurableData"]["host"]["password"]
+            host.port = self.services["configurableData"]["host"]["port"]
+
+            try:
+                state = get_process_status(
+                    host.ipaddress,
+                    host.port,
+                    host.user,
+                    host.passwd,
+                    router.linklocalip,
+                    "ip addr | grep eth6 | grep state | awk '{print $9;}'")
+
+                mac = get_process_status(
+                    host.ipaddress,
+                    host.port,
+                    host.user,
+                    host.passwd,
+                    router.linklocalip,
+                    "ip addr | grep link/ether | awk '{print $2;}' | sed -n 7p")
+            except KeyError:
+                self.skipTest("Provide a marvin config file with host credentials to run %s" % self._testMethodName)
+
+            self.logger.debug("Result from the Router on IP '%s' is -> state: '%s', mac: '%s'" % (router.linklocalip, state, mac))
+            state_holder[router.linklocalip]["state"] = str(state)
+            state_holder[router.linklocalip]["mac"] = str(mac)
+
+        check_state = state_holder[routers[0].linklocalip]["state"].count(state_holder[routers[1].linklocalip]["state"])
+        check_mac = state_holder[routers[0].linklocalip]["mac"].count(state_holder[routers[1].linklocalip]["mac"])
+
+        self.assertTrue(check_state == 0, "Routers private gateway interface should not be on the same state!")
+        self.assertTrue(check_mac == 0, "Routers private gateway interface should not have the same mac address!")
+
+    def check_routers_state(self, routers, status_to_check="MASTER", expected_count=1):
+        vals = ["MASTER", "BACKUP", "UNKNOWN"]
+        cnts = [0, 0, 0]
+
+        result = "UNKNOWN"
+        for router in routers:
+            if router.state == "Running":
+                hosts = list_hosts(
+                    self.apiclient,
+                    zoneid=router.zoneid,
+                    type='Routing',
+                    state='Up',
+                    id=router.hostid
+                )
+                self.assertEqual(
+                    isinstance(hosts, list),
+                    True,
+                    "Check list host returns a valid list"
+                )
+                host = hosts[0]
+
+                if self.hypervisor.lower() in ('vmware', 'hyperv'):
+                        result = str(get_process_status(
+                            self.apiclient.connection.mgtSvr,
+                            22,
+                            self.apiclient.connection.user,
+                            self.apiclient.connection.passwd,
+                            router.linklocalip,
+                            "sh /opt/cloud/bin/checkrouter.sh ",
+                            hypervisor=self.hypervisor
+                        ))
+                else:
+                    try:
+                        host.user, host.passwd = get_host_credentials(
+                            self.config, host.ipaddress)
+                        result = str(get_process_status(
+                            host.ipaddress,
+                            22,
+                            host.user,
+                            host.passwd,
+                            router.linklocalip,
+                            "sh /opt/cloud/bin/checkrouter.sh "
+                        ))
+
+                    except KeyError:
+                        self.skipTest(
+                            "Marvin configuration has no host credentials to\
+                                    check router services")
+
+                if result.count(status_to_check) == 1:
+                    cnts[vals.index(status_to_check)] += 1
+
+        if cnts[vals.index(status_to_check)] != expected_count:
+            self.fail("Expected '%s' routers at state '%s', but found '%s'!" % (expected_count, status_to_check, cnts[vals.index(status_to_check)]))