[mgs-updates] RotCommunicationFailed should be transient (#8930)

Closes: #8886

Author: karencfv

nexus/mgs-updates/src/driver_update.rs

@@ -727,6 +727,7 @@ async fn wait_for_update_done(
             // * non-empty transient_boot_preference (RoT only)
             // * failure to fetch inventory from sled-agent (host OS only)
             // * failure to determine an active slot artifact
+            // * failure to communicate with the RoT
             //
             // We have no reason to think these won't converge, so we proceed
             // with waiting.
@@ -742,6 +743,7 @@ async fn wait_for_update_done(
             | Err(PrecheckError::MismatchedHostOsActiveSlot { .. })
             | Err(PrecheckError::DeterminingActiveArtifact { .. })
             | Err(PrecheckError::DeterminingHostOsBootDisk { .. })
+            | Err(PrecheckError::RotCommunicationFailed { .. })
             | Ok(PrecheckStatus::ReadyForUpdate) => {
                 if before.elapsed() >= timeout {
                     return Err(UpdateWaitError::Timeout(timeout));
@@ -756,8 +758,7 @@ async fn wait_for_update_done(
                 | PrecheckError::WrongActiveVersion { .. }
                 | PrecheckError::WrongActiveArtifact { .. }
                 | PrecheckError::WrongHostOsBootDisk { .. }
-                | PrecheckError::InvalidHostPhase1Slot { .. }
-                | PrecheckError::RotCommunicationFailed { .. }),
+                | PrecheckError::InvalidHostPhase1Slot { .. }),
             ) => {
                 // Stop trying to make this update happen.  It's not going to
                 // happen.