44from datetime import datetime , timedelta
55import re
66from unicon .eal .dialogs import Dialog
7- from unicon .core .errors import SubCommandFailure
7+ from unicon .core .errors import SubCommandFailure , StateMachineError
88from unicon .bases .routers .services import BaseService
99
10+ from .exception import StackMemberReadyException
1011from .utils import StackUtils
1112from unicon .plugins .generic .statements import custom_auth_statements , buffer_settled
1213from unicon .plugins .generic .service_statements import standby_reset_rp_statement_list
@@ -113,6 +114,8 @@ def call_service(self, command=None,
113114 connect_dialog = self .connection .connection_provider .get_connection_dialog ()
114115 dialog += connect_dialog
115116
117+ start_time = datetime .now ()
118+
116119 conn .log .info ('Processing on active rp %s-%s' % (conn .hostname , conn .alias ))
117120 conn .sendline (switchover_cmd )
118121 try :
@@ -141,10 +144,11 @@ def call_service(self, command=None,
141144 sleep (self .connection .settings .POST_SWITCHOVER_SLEEP )
142145
143146 # check all members are ready
144- conn .state_machine .detect_state (conn .spawn , context = conn .context )
147+ recheck_sleep_interval = self .connection .settings .SWITCHOVER_POSTCHECK_INTERVAL
148+ recheck_max = timeout - (datetime .now () - start_time ).seconds
145149
146- interval = self .connection .settings . SWITCHOVER_POSTCHECK_INTERVAL
147- if utils .is_all_member_ready (conn , timeout = timeout , interval = interval ):
150+ self .connection .log . info ( 'Wait for all members to be ready.' )
151+ if utils .is_all_member_ready (conn , timeout = recheck_max , interval = recheck_sleep_interval ):
148152 self .connection .log .info ('All members are ready.' )
149153 else :
150154 self .connection .log .info ('Timeout in %s secs. '
@@ -245,9 +249,10 @@ def call_service(self,
245249
246250 reload_dialog += Dialog ([switch_prompt ])
247251
252+ conn .context ['post_reload_timeout' ] = timedelta (seconds = self .post_reload_wait_time )
253+
248254 conn .log .info ('Processing on active rp %s-%s' % (conn .hostname , conn .alias ))
249255 start_time = current_time = datetime .now ()
250- timeout_time = timedelta (seconds = timeout )
251256 conn .sendline (reload_cmd )
252257 try :
253258 reload_cmd_output = reload_dialog .process (conn .spawn ,
@@ -256,6 +261,9 @@ def call_service(self,
256261 context = conn .context )
257262 self .result = reload_cmd_output .match_output
258263 self .get_service_result ()
264+ except StackMemberReadyException as e :
265+ conn .log .debug ('This is an expected exception for getting out of the dialog process' )
266+ pass
259267 except Exception as e :
260268 raise SubCommandFailure ('Error during reload' , e ) from e
261269
@@ -273,59 +281,45 @@ def call_service(self,
273281 for subconn in self .connection .subconnections :
274282 self .connection .log .info ('Processing on rp '
275283 '%s-%s' % (conn .hostname , subconn .alias ))
284+ subconn .context ['post_reload_timeout' ] = timedelta (seconds = self .post_reload_wait_time )
276285 utils .boot_process (subconn , timeout , self .prompt_recovery , reload_dialog )
277286
278287 except Exception as e :
279288 self .connection .log .error (e )
280289 raise SubCommandFailure ('Reload failed.' , e ) from e
281290 else :
282- conn .log .info ('Waiting for boot messages to settle for {} seconds' .format (
283- self .post_reload_wait_time
284- ))
285- wait_time = timedelta (seconds = self .post_reload_wait_time )
286- settle_time = current_time = datetime .now ()
287- while (current_time - settle_time ) < wait_time :
288- if buffer_settled (conn .spawn , self .post_reload_wait_time ):
289- conn .log .info ('Buffer settled, accessing device..' )
290- break
291- current_time = datetime .now ()
292- if (current_time - start_time ) > timeout_time :
293- conn .log .info ('Time out, trying to acces device..' )
294- break
295- try :
296- # bring device to enable mode
297- conn .state_machine .go_to ('any' , conn .spawn , timeout = timeout ,
298- prompt_recovery = self .prompt_recovery ,
299- context = conn .context )
300- conn .state_machine .go_to ('enable' , conn .spawn , timeout = timeout ,
301- prompt_recovery = self .prompt_recovery ,
302- context = conn .context )
303- except Exception as e :
304- raise SubCommandFailure ('Failed to bring device to disable mode.' , e ) from e
291+ self .connection .log .info ('Processing autoboot on rp %s-%s' % (conn .hostname , conn .alias ))
292+
293+
294+ self .connection .log .info ('Sleeping for %s secs.' % \
295+ self .connection .settings .STACK_POST_RELOAD_SLEEP )
296+ sleep (self .connection .settings .STACK_POST_RELOAD_SLEEP )
297+
298+ # make sure detect_state is good to reduce the chance of timeout later
299+ recheck_sleep_interval = self .connection .settings .RELOAD_POSTCHECK_INTERVAL
300+ recheck_max = timeout - (datetime .now () - start_time ).seconds
301+
305302 # check active and standby rp is ready
306303 self .connection .log .info ('Wait for Standby RP to be ready.' )
307304
308- interval = self .connection .settings .RELOAD_POSTCHECK_INTERVAL
309- if utils .is_active_standby_ready (conn , timeout = timeout , interval = interval ):
305+ if utils .is_active_standby_ready (conn , timeout = recheck_max , interval = recheck_sleep_interval ):
310306 self .connection .log .info ('Active and Standby RPs are ready.' )
311307 else :
312308 self .connection .log .info ('Timeout in %s secs. '
313309 'Standby RP is not in Ready state. Reload failed' % timeout )
314310 self .result = False
315311 return
312+
313+ self .connection .log .info ('Start checking state of all members' )
314+ recheck_max = timeout - (datetime .now () - start_time ).seconds
315+ if utils .is_all_member_ready (conn , timeout = recheck_max , interval = recheck_sleep_interval ):
316+ self .connection .log .info ('All Members are ready.' )
317+ else :
318+ self .connection .log .info (f'Timeout in { recheck_max } secs. '
319+ f'Not all members are in Ready state. Reload failed' )
320+ self .result = False
321+ return
316322
317- if member :
318- if utils .is_all_member_ready (conn , timeout = timeout , interval = interval ):
319- self .connection .log .info ('All Members are ready.' )
320- else :
321- self .connection .log .info (f'Timeout in { timeout } secs. '
322- f'Member{ member } is not in Ready state. Reload failed' )
323- self .result = False
324- return
325-
326- self .connection .log .info ('Sleeping for %s secs.' % \
327- self .connection .settings .STACK_POST_RELOAD_SLEEP )
328- sleep (self .connection .settings .STACK_POST_RELOAD_SLEEP )
329323
330324 self .connection .log .info ('Disconnecting and reconnecting' )
331325 self .connection .disconnect ()
@@ -578,10 +572,12 @@ def _check_invalid_mac(con):
578572 return True
579573 return False
580574
581- from genie .utils .timeout import Timeout
582- exec_timeout = Timeout (timeout , 15 )
575+ chk_interval = con .settings .RELOAD_POSTCHECK_INTERVAL
583576 found_invalid_mac = False
584- while exec_timeout .iterate ():
577+ start_time2 = time ()
578+ while (time () - start_time2 ) < timeout :
579+ t_left = timeout - (time () - start_time2 )
580+ con .log .info ('-- checking time left: %0.1f secs' % t_left )
585581 con .log .info ('Make sure no invalid mac address 0000.0000.0000' )
586582 if not _check_invalid_mac (con ):
587583 con .log .info ('Did not find invalid mac as 0000.0000.0000' )
@@ -590,7 +586,8 @@ def _check_invalid_mac(con):
590586 else :
591587 con .log .warning ('Found 0000.0000.0000 mac address' )
592588 found_invalid_mac = True
593- exec_timeout .sleep ()
589+ con .log .info (f'Sleep { chk_interval } secs' )
590+ sleep (chk_interval )
594591 continue
595592 else :
596593 if found_invalid_mac :
0 commit comments