Expose all Progressing messages

Author: hongkailiu

pkg/monitor/monitorapi/types.go

@@ -376,6 +376,7 @@ const (
 	SourcePathologicalEventMarker IntervalSource = "PathologicalEventMarker" // not sure if this is really helpful since the events all have a different origin
 	SourceClusterOperatorMonitor  IntervalSource = "ClusterOperatorMonitor"
 	SourceOperatorState           IntervalSource = "OperatorState"
+	SourceVersionState            IntervalSource = "VersionState"
 	SourceNodeState                              = "NodeState"
 	SourcePodState                               = "PodState"
 	SourceCloudMetrics                           = "CloudMetrics"

pkg/monitortests/clusterversionoperator/legacycvomonitortests/monitortest.go

@@ -2,13 +2,16 @@ package legacycvomonitortests
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/openshift/origin/pkg/monitortestframework"
 
 	"github.com/openshift/origin/pkg/monitor/monitorapi"
 	"github.com/openshift/origin/pkg/monitortestlibrary/platformidentification"
 	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
+
+	configv1 "github.com/openshift/api/config/v1"
 	"k8s.io/client-go/rest"
 )
 
@@ -34,7 +37,50 @@ func (w *legacyMonitorTests) CollectData(ctx context.Context, storageDir string,
 }
 
 func (*legacyMonitorTests) ConstructComputedIntervals(ctx context.Context, startingIntervals monitorapi.Intervals, recordedResources monitorapi.ResourcesMap, beginning, end time.Time) (monitorapi.Intervals, error) {
-	return nil, nil
+	ret := monitorapi.Intervals{}
+	ret = append(ret, intervalsFromEventsClusterVersionProgressing(startingIntervals)...)
+	return ret, nil
+}
+
+var lastClusterVersionProgressingInterval *monitorapi.Interval
+
+func intervalsFromEventsClusterVersionProgressing(intervals monitorapi.Intervals) monitorapi.Intervals {
+	var ret monitorapi.Intervals
+
+	for _, event := range intervals {
+		if event.Source != monitorapi.SourceClusterOperatorMonitor {
+			continue
+		}
+		if cvName := event.Locator.Keys[monitorapi.LocatorClusterVersionKey]; cvName != "version" {
+			continue
+		}
+		currentCondition := monitorapi.GetOperatorConditionStatus(event)
+		if currentCondition == nil {
+			continue
+		}
+
+		if currentCondition.Type != configv1.OperatorProgressing {
+			continue
+		}
+
+		if lastClusterVersionProgressingInterval != nil {
+			ret = append(ret, monitorapi.NewInterval(monitorapi.SourceVersionState, monitorapi.Warning).
+				Locator(event.Locator).
+				Message(monitorapi.NewMessage().Reason(lastClusterVersionProgressingInterval.Message.Reason).
+					HumanMessage(strings.Replace(lastClusterVersionProgressingInterval.Message.HumanMessage, "changed to ", "stayed in ", 1)).
+					WithAnnotation(monitorapi.AnnotationCondition, string(configv1.OperatorProgressing)).
+					WithAnnotation(monitorapi.AnnotationStatus, string(configv1.ConditionTrue))).
+				Build(lastClusterVersionProgressingInterval.From, event.From),
+			)
+			lastClusterVersionProgressingInterval = nil
+		}
+
+		if currentCondition.Status == configv1.ConditionTrue &&
+			strings.Contains(event.Message.HumanMessage, ProgressingWaitingCOsKey) {
+			lastClusterVersionProgressingInterval = event.DeepCopy()
+		}
+	}
+	return ret
 }
 
 func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.Context, finalIntervals monitorapi.Intervals) ([]*junitapi.JUnitTestCase, error) {

pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go

@@ -867,7 +867,7 @@ func updateCOWaiting(interval monitorapi.Interval, waiting map[string]monitorapi
 	if waiting == nil {
 		return
 	}
-	if interval.Source != monitorapi.SourceOperatorState ||
+	if interval.Source != monitorapi.SourceVersionState ||
 		interval.Locator.Type != monitorapi.LocatorTypeClusterVersion ||
 		interval.Locator.Keys[monitorapi.LocatorClusterVersionKey] != "version" {
 		return

pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators_test.go

@@ -230,14 +230,13 @@ func Test_updateCOWaiting(t *testing.T) {
 	now := time.Now()
 	next := 3 * time.Hour
 	interval := func(m string, start time.Time, d time.Duration) monitorapi.Interval {
-		return monitorapi.NewInterval(monitorapi.SourceOperatorState, monitorapi.Warning).
+		return monitorapi.NewInterval("foo", monitorapi.Warning).
 			Locator(monitorapi.NewLocator().ClusterVersion(&configv1.ClusterVersion{
-				ObjectMeta: metav1.ObjectMeta{Name: "version"}})).
+				ObjectMeta: metav1.ObjectMeta{Name: "bar"}})).
 			Message(monitorapi.NewMessage().Reason("reason").
 				HumanMessage(m).
 				WithAnnotation(monitorapi.AnnotationCondition, string(configv1.OperatorProgressing)).
 				WithAnnotation(monitorapi.AnnotationStatus, string(configv1.ConditionTrue))).
-			Display().
 			Build(start, start.Add(d))
 	}
 
@@ -277,14 +276,13 @@ func Test_updateCOWaiting(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		i := monitorapi.NewInterval(monitorapi.SourceOperatorState, monitorapi.Warning).
+		i := monitorapi.NewInterval(monitorapi.SourceVersionState, monitorapi.Warning).
 			Locator(monitorapi.NewLocator().ClusterVersion(&configv1.ClusterVersion{
 				ObjectMeta: metav1.ObjectMeta{Name: "version"}})).
 			Message(monitorapi.NewMessage().Reason("reason").
 				HumanMessage(tt.message).
 				WithAnnotation(monitorapi.AnnotationCondition, string(configv1.OperatorProgressing)).
 				WithAnnotation(monitorapi.AnnotationStatus, string(configv1.ConditionTrue))).
-			Display().
 			Build(now.Add(next), now.Add(next+tt.d))
 		t.Run(tt.name, func(t *testing.T) {
 			updateCOWaiting(i, tt.waiting)

pkg/monitortests/testframework/watchclusteroperators/operator.go

@@ -176,7 +176,11 @@ func startClusterOperatorMonitoring(ctx context.Context, m monitorapi.RecorderWr
 			for i := range cv.Status.Conditions {
 				s := &cv.Status.Conditions[i]
 				previous := findOperatorStatusCondition(oldCV.Status.Conditions, s.Type)
-				if previous == nil || s.Status != previous.Status {
+				if previous == nil || s.Status != previous.Status ||
+					// Expose all messages for Progressing=True and Failing=Unknown
+					// There are tests replying on them
+					(s.Type == configv1.OperatorProgressing && s.Status == configv1.ConditionTrue && s.Message != previous.Message) ||
+					(s.Type == "Failing" && s.Status == configv1.ConditionUnknown && s.Message != previous.Message) {
 					level := monitorapi.Warning
 					if s.Type == configv1.OperatorDegraded && s.Status == configv1.ConditionTrue {
 						level = monitorapi.Error