Retry netlbfo team creation

The WMI implementation for NETLBFO is sometimes flaky and does not fail
when a bond cannot be created.

Added 5 retries for the team creation, just in case the WMI call to
create the bond did not fail, but the bond was not created.

In this scenario, the _wait_for_nic method is called and it fails,
as it does not find the team nic and the retry is performed.

Updated the wait for team nic retry interval to 10 seconds,
as the team nic may be retrieved with WMI after a longer time.

Change-Id: I7622a4a30b867f335ed8f3288ca2fea830daf94b

Author: ader1990

cloudbaseinit/tests/utils/windows/test_netlbfo.py

@@ -41,6 +41,7 @@ def setUp(self):
     def tearDown(self):
         self._module_patcher.stop()
 
+    @mock.patch('time.sleep')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._get_primary_adapter_name')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._create_team')
     @mock.patch(MODPATH + '.NetLBFOTeamManager._add_team_member')
@@ -49,10 +50,11 @@ def tearDown(self):
     @mock.patch(MODPATH + '.NetLBFOTeamManager.delete_team')
     def _test_create_team(self, mock_delete_team, mock_wait_for_nic,
                           mock_set_primary_nic_vlan_id, mock_add_team_member,
-                          mock_create_team, mock_get_primary_adapter_name,
-                          mode_not_found=False, lb_algo_not_found=False,
+                          mock_create_team, mock_primary_adapter_name,
+                          mock_time_sleep, mode_not_found=False,
+                          lb_algo_not_found=False,
                           add_team_member_fail=False):
-        mock_get_primary_adapter_name.return_value = mock.sentinel.pri_nic_name
+        mock_primary_adapter_name.return_value = mock.sentinel.pri_nic_name
         mock_create_team.return_value = None
 
         lacp_timer = network_model.BOND_LACP_RATE_FAST
@@ -100,9 +102,6 @@ def _test_create_team(self, mock_delete_team, mock_wait_for_nic,
                 mock.sentinel.mac, mock.sentinel.pri_nic_name,
                 mock.sentinel.vlan_id, lacp_timer)
 
-        mock_add_team_member.assert_called_once_with(
-            conn, mock.sentinel.team_name, mock.sentinel.other_member)
-
         if not add_team_member_fail:
             mock_set_primary_nic_vlan_id.assert_called_once_with(
                 conn, mock.sentinel.team_name, mock.sentinel.vlan_id)
@@ -111,8 +110,14 @@ def _test_create_team(self, mock_delete_team, mock_wait_for_nic,
                 2, 3, mock.sentinel.pri_nic_name, 1)
             mock_wait_for_nic.assert_called_once_with(
                 mock_team_nic.Name)
+            mock_add_team_member.assert_called_once_with(
+                conn, mock.sentinel.team_name, mock.sentinel.other_member)
         else:
-            mock_delete_team.assert_called_once_with(mock.sentinel.team_name)
+            mock_add_team_member.assert_called_with(
+                conn, mock.sentinel.team_name, mock.sentinel.other_member)
+            mock_delete_team.assert_called_with(mock.sentinel.team_name)
+            self.assertEqual(mock_add_team_member.call_count, 6)
+            self.assertEqual(mock_delete_team.call_count, 6)
 
     def test_create_team(self):
         self._test_create_team()

cloudbaseinit/utils/windows/netlbfo.py

@@ -132,6 +132,7 @@ def _create_team(conn, team_name, nic_name, teaming_mode, lb_algo,
         operation_options = {u'custom_options': custom_options}
         team.put(operation_options=operation_options)
 
+    @retry_decorator.retry_decorator(max_retry_count=5)
     def create_team(self, team_name, mode, load_balancing_algorithm,
                     members, mac_address, primary_nic_name=None,
                     primary_nic_vlan_id=None, lacp_timer=None):
@@ -178,7 +179,8 @@ def create_team(self, team_name, mode, load_balancing_algorithm,
             raise ex
 
     @staticmethod
-    @retry_decorator.retry_decorator(max_retry_count=10)
+    @retry_decorator.retry_decorator(max_retry_count=10,
+                                     max_sleep_time=10)
     def _wait_for_nic(nic_name):
         conn = wmi.WMI(moniker='//./root/cimv2')
         if not conn.Win32_NetworkAdapter(NetConnectionID=nic_name):