test: fix the autobursting detection (#1443)

Bursting on Autopilot requires clusters to meet the following conditions:
- You originally created the cluster with GKE version 1.26 or later.
- The cluster is running GKE version 1.30.2-gke.1394000 or later.

When upgrading an Autopilot cluster to a supported version, GKE upgrades
the worker nodes to match the control plane version over time. A control
plane restart is required to enable bursting, and must happen after all
the nodes run a supported version.

There isn't a good way to check the control plane restart, this commit
only guarantees that all node versions are 1.30.2-gke.1394000 or later
when nt.ClusterSupportsBursting is true.

Link: https://cloud.google.com/kubernetes-engine/docs/how-to/pod-bursting-gke#availability-in-gke

b/369006133
b/370402111

Author: nan-yu

e2e/nomostest/new.go

@@ -183,7 +183,7 @@ func SharedTestEnv(t nomostesting.NTB, opts *ntopts.New) *NT {
 		OCIClient:               sharedNt.OCIClient,
 	}
 
-	t.Logf("using shared test env: %s, cluster version: %s, cluster hash: %s", sharedNt.ClusterName, nt.ClusterVersion, nt.ClusterHash)
+	t.Logf("using shared test env: %s, cluster version: %s, cluster hash: %s, support bursting: %t", nt.ClusterName, nt.ClusterVersion, nt.ClusterHash, nt.ClusterSupportsBursting)
 
 	if opts.SkipConfigSyncInstall {
 		return nt

e2e/nomostest/nt.go

@@ -906,10 +906,27 @@ func (nt *NT) autopilotClusterSupportsBursting() (bool, error) {
 		return false, err
 	}
 	minInitialVersion := clusterversion.ClusterVersion{Major: 1, Minor: 26}
-	minCurrentVersion := clusterversion.ClusterVersion{
-		Major: 1, Minor: 30, Patch: 2, Suffix: "-gke.1394000"}
-	return initialClusterVersion.IsAtLeast(minInitialVersion) &&
-		nt.ClusterVersion.IsAtLeast(minCurrentVersion), nil
+	if !initialClusterVersion.IsAtLeast(minInitialVersion) {
+		return false, nil
+	}
+
+	minCurrentVersion := clusterversion.ClusterVersion{Major: 1, Minor: 30, Patch: 2, Suffix: "-gke.1394000"}
+	args = []string{"get", "nodes", "-o", `jsonpath={.items[*].status.nodeInfo.kubeletVersion}`}
+	out, err = nt.Shell.Kubectl(args...)
+	if err != nil {
+		return false, err
+	}
+	versions := strings.Split(strings.TrimSpace(string(out)), " ")
+	for _, nodeVersion := range versions {
+		v, err := clusterversion.ParseClusterVersion(nodeVersion)
+		if err != nil {
+			return false, err
+		}
+		if !v.IsAtLeast(minCurrentVersion) {
+			return false, nil
+		}
+	}
+	return true, nil
 }
 
 func (nt *NT) detectClusterSupportsBursting() {