Merge pull request #30154 from hongkailiu/OTA-1601

OTA-1601: upgrade status: recover checking the updating COs

Author: openshift-merge-bot[bot]

pkg/monitortests/cli/adm_upgrade/status/controlplane.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	operatorLinePattern = regexp.MustCompile(`^\S+\s+\S+\s+\S+\s+.*$`)
+	operatorFirstLinePattern = regexp.MustCompile(`^\S+\s+\S+\s+\S+(\s+.*)?$`)
 )
 
 func (w *monitor) controlPlane() *junitapi.JUnitTestCase {
@@ -100,20 +100,18 @@ func (w *monitor) controlPlane() *junitapi.JUnitTestCase {
 			}
 
 			items := len(strings.Split(updatingOperators, ","))
-			// TODO: These should actually exactly match, but `oc adm upgrade status` emits operators with linebreaks in
-			// messages in a crappy way which we will need to fix
-			if len(cp.Operators) < items {
+			if len(cp.Operators) != items {
 				fail(fmt.Sprintf("Control plane summary contains Updating key with %d operators but operators section has %d items", items, len(cp.Operators)))
 				continue
 			}
 		}
 
-		// TODO: `oc adm upgrade status` emits operators with linebreaks in messages in a crappy way which we will need to fix
-		// for _, operator := range cp.Operators {
-		// 	if !operatorLinePattern.MatchString(operator) {
-		// 		fail(fmt.Sprintf("Bad line in operators: %s", operator))
-		// 	}
-		// }
+		for _, operator := range cp.Operators {
+			firstLine := strings.Split(operator, "\n")[0]
+			if !operatorFirstLinePattern.MatchString(firstLine) {
+				fail(fmt.Sprintf("Bad first line in operators: %s", operator))
+			}
+		}
 
 		for _, node := range cp.Nodes {
 			if !nodeLinePattern.MatchString(node) {

pkg/monitortests/cli/adm_upgrade/status/controlplane_test.go

@@ -94,8 +94,6 @@ ip-10-0-111-19.us-west-1.compute.internal   Outdated     Pending   4.20.0-0.ci-2
 SINCE   LEVEL   IMPACT   MESSAGE
 8m57s   Info    None     Update is proceeding well`
 
-// TODO: This is a bug in `oc adm upgrade status` that we will fix but for now we need to tolerate
-// Eventually we will fail on output like this and we will also need to add a testcase for the fixed output
 var operatorsWithLinebreaksInMessages = `Unable to fetch alerts, ignoring alerts in 'Update Health':  failed to get alerts from Thanos: no token is currently in use for this session
 = Control Plane =
 Assessment:      Progressing
@@ -106,14 +104,14 @@ Duration:        24m (Est. Time Remaining: 45m)
 Operator Health: 34 Healthy
 
 Updating Cluster Operators
-NAME             SINCE   REASON                                            MESSAGE
-image-registry   6s      DeploymentNotCompleted::NodeCADaemonUnavailable   NodeCADaemonProgressing: The daemon set node-ca is deploying node pods
-Progressing: The deployment has not completed
-monitoring                     4s    RollOutInProgress                                                                Rolling out the stack.
-openshift-controller-manager   11s   RouteControllerManager_DesiredStateNotYetAchieved::_DesiredStateNotYetAchieved   Progressing: deployment/controller-manager: observed generation is 10, desired generation is 11
-Progressing: deployment/controller-manager: updated replicas is 1, desired replicas is 3
-RouteControllerManagerProgressing: deployment/route-controller-manager: observed generation is 7, desired generation is 8
-RouteControllerManagerProgressing: deployment/route-controller-manager: updated replicas is 1, desired replicas is 3
+NAME                           SINCE   REASON               MESSAGE
+image-registry                 6s      DeploymentNotCo...   NodeCADaemonProgressing: The daemon set node-ca is deploying node pods
+                                                            Progressing: The deployment has not completed
+monitoring                     4s      RollOutInProgre...   Rolling out the stack.
+openshift-controller-manager   11s     RouteController...   Progressing: deployment/controller-manager: observed generation is 10, desired generation is 11
+                                                            Progressing: deployment/controller-manager: updated replicas is 1, desired replicas is 3
+                                                            RouteControllerManagerProgressing: deployment/route-controller-manager: observed generation is 7, desired generation is 8
+                                                            RouteControllerManagerProgressing: deployment/route-controller-manager: updated replicas is 1, desired replicas is 3
 
 Control Plane Nodes
 NAME                          ASSESSMENT   PHASE     VERSION                                                     EST   MESSAGE
@@ -136,6 +134,28 @@ ip-10-0-64-121.ec2.internal   Outdated     Pending   4.20.0-0.ci-2025-08-13-1748
 SINCE    LEVEL   IMPACT   MESSAGE
 24m12s   Info    None     Update is proceeding well`
 
+var operatorsWithEmptyMessage = `Unable to fetch alerts, ignoring alerts in 'Update Health':  failed to get alerts from Thanos: no token is currently in use for this session
+= Control Plane =
+Assessment:      Progressing
+Target Version:  4.20.0-0.ci-2025-08-26-154705-test-ci-op-jy9dt0lx-latest (from 4.20.0-0.ci-2025-08-26-145530-test-ci-op-jy9dt0lx-initial)
+Updating:        authentication
+Completion:      9% (3 operators updated, 1 updating, 30 waiting)
+Duration:        7m27s (Est. Time Remaining: 36m)
+Operator Health: 32 Healthy, 1 Unavailable, 1 Available but degraded
+
+Updating Cluster Operators
+NAME             SINCE   REASON                 MESSAGE
+authentication   58s     WellKnownReadyCon...   
+
+Control Plane Nodes
+NAME                                       ASSESSMENT   PHASE     VERSION                                                     EST   MESSAGE
+ip-10-0-35-68.us-east-2.compute.internal   Outdated     Pending   4.20.0-0.ci-2025-08-26-145530-test-ci-op-jy9dt0lx-initial   ?     
+
+= Update Health =
+SINCE   LEVEL   IMPACT   MESSAGE
+7m27s   Info    None     Update is proceeding well
+`
+
 var controlPlaneCompleted = `Unable to fetch alerts, ignoring alerts in 'Update Health':  failed to get alerts from Thanos: no token is currently in use for this session
 = Control Plane =
 Update to 4.20.0-0.ci-2025-08-13-182454-test-ci-op-5wilvz46-latest successfully completed at 2025-08-13T20:33:32Z (duration: 59m)
@@ -238,6 +258,15 @@ func TestMonitor_ControlPlane(t *testing.T) {
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status control plane section is consistent",
 			},
 		},
+		{
+			name: "operators section with empty message",
+			snapshots: []snapshot{
+				{when: time.Now(), out: operatorsWithEmptyMessage},
+			},
+			expected: &junitapi.JUnitTestCase{
+				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status control plane section is consistent",
+			},
+		},
 		{
 			name: "control plane completed",
 			snapshots: []snapshot{

pkg/monitortests/cli/adm_upgrade/status/outputmodel.go

@@ -260,12 +260,17 @@ func (p *parser) parseControlPlaneOperators() ([]string, error) {
 	var operators []string
 
 	for {
-		line, done := p.next()
+		line, done := p.next(preserveLeadingWhitespace)
 		if done || line == "" {
 			break
 		}
-
-		operators = append(operators, line)
+		trimLeft := strings.TrimLeft(line, " \t")
+		if len(operators) == 0 || line == trimLeft {
+			operators = append(operators, line)
+		} else {
+			last := len(operators) - 1
+			operators[last] = fmt.Sprintf("%s\n%s", operators[last], trimLeft)
+		}
 	}
 
 	if len(operators) == 0 {