test(node): add nested GuestOS upgrade orchestrator check and diagnostics (#7498)

NODE-1761

The nested GuestOS upgrade tests only check for the final GuestOS
version—not the Orchestrator health. This is why we haven’t seen
flakiness on these tests, but have on the HostOS upgrade tests. This
adds the orchestrator check to the nested GuestOS upgrade tests along
with diagnostic logging in failures.

Author: andrewbattat

rs/tests/nested/guestos_upgrade.rs

@@ -13,8 +13,8 @@ use nested::{HOST_VM_NAME, registration};
 
 use nested::util::{
     NODE_UPGRADE_BACKOFF, NODE_UPGRADE_TIMEOUT, elect_guestos_version,
-    get_blessed_guestos_versions, get_unassigned_nodes_config, update_unassigned_nodes,
-    wait_for_expected_guest_version,
+    get_blessed_guestos_versions, get_unassigned_nodes_config, try_logging_guestos_diagnostics,
+    update_unassigned_nodes, wait_for_expected_guest_version,
 };
 
 fn main() -> Result<()> {
@@ -48,9 +48,11 @@ pub fn upgrade_guestos(env: TestEnv) {
 
     registration(env.clone());
 
-    let guest_ipv6 = env
+    let host = env
         .get_nested_vm(HOST_VM_NAME)
-        .expect("Unable to find HostOS node.")
+        .expect("Unable to find HostOS node.");
+
+    let guest_ipv6 = host
         .get_nested_network()
         .expect("Unable to get nested network")
         .guest_ip;
@@ -128,5 +130,11 @@ pub fn upgrade_guestos(env: TestEnv) {
         )
         .await
         .expect("guest failed to upgrade");
+
+        info!(logger, "Waiting for Orchestrator dashboard...");
+        if let Err(e) = host.await_orchestrator_dashboard_accessible() {
+            try_logging_guestos_diagnostics(&host, &logger);
+            panic!("Orchestrator dashboard is not accessible: {e}");
+        }
     });
 }