Record OperatorPolicy history in status

Refs:
 - https://issues.redhat.com/browse/ACM-20803

Signed-off-by: Justin Kulikauskas <[email protected]>

Author: JustinKuli

api/v1beta1/operatorpolicy_types.go

@@ -242,6 +242,10 @@ type OperatorPolicyStatus struct {
 	// policy is waiting for OLM to resolve the situation. If in the recent past,
 	// the policy may update the status of the Subscription.
 	SubscriptionInterventionTime *metav1.Time `json:"subscriptionInterventionTime,omitempty"`
+
+	// History is a list of the most recent compliance messages for this operator policy.
+	// The first entry is the most recent, and the list is limited to 10 entries.
+	History []policyv1.HistoryEvent `json:"history,omitempty"`
 }
 
 // RelatedObjsOfKind iterates over the related objects in the status and returns a map of the index

api/v1beta1/zz_generated.deepcopy.go

@@ -267,18 +267,12 @@ func (r *OperatorPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 
 	errs := make([]error, 0)
 
-	conditionsToEmit, conditionChanged, err := r.handleResources(ctx, policy)
+	conditionsToEmit, statusChanged, err := r.handleResources(ctx, policy)
 	if err != nil {
 		errs = append(errs, err)
 	}
 
-	if conditionChanged || !reflect.DeepEqual(policy.Status, originalStatus) {
-		if err := r.Status().Update(ctx, policy); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	if conditionChanged {
+	if statusChanged {
 		// Add an event for the "final" state of the policy, otherwise this only has the
 		// "early" events (and possibly has zero events).
 		conditionsToEmit = append(conditionsToEmit, calculateComplianceCondition(policy))
@@ -288,6 +282,33 @@ func (r *OperatorPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		if err := r.emitComplianceEvent(ctx, policy, cond); err != nil {
 			errs = append(errs, err)
 		}
+
+		var latestEvent policyv1.HistoryEvent
+
+		if len(policy.Status.History) > 0 {
+			latestEvent = policy.Status.History[0]
+		}
+
+		if latestEvent.Message != cond.Message {
+			statusChanged = true
+			newEvent := policyv1.HistoryEvent{
+				LastTimestamp: metav1.MicroTime(cond.LastTransitionTime),
+				Message:       cond.Message,
+			}
+
+			policy.Status.History = append([]policyv1.HistoryEvent{newEvent}, policy.Status.History...)
+
+			maxHistoryLength := 10 // At some point, we may make this length configurable
+			if len(policy.Status.History) > maxHistoryLength {
+				policy.Status.History = policy.Status.History[:maxHistoryLength]
+			}
+		}
+	}
+
+	if statusChanged || !reflect.DeepEqual(policy.Status, originalStatus) {
+		if err := r.Status().Update(ctx, policy); err != nil {
+			errs = append(errs, err)
+		}
 	}
 
 	result := reconcile.Result{}

controllers/operatorpolicy_controller.go

@@ -6,7 +6,6 @@ import (
 	"slices"
 	"sort"
 	"strings"
-	"time"
 
 	operatorv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
@@ -302,13 +301,27 @@ func calculateComplianceCondition(policy *policyv1beta1.OperatorPolicy) metav1.C
 		}
 	}
 
+	message := strings.Join(messages, ", ")
+
+	if foundNonCompliant {
+		message = "NonCompliant; " + message
+	} else {
+		message = "Compliant; " + message
+	}
+
+	maxMessageLength := 4096
+	if len(message) > maxMessageLength {
+		truncMsg := "...[truncated]"
+		message = message[:(maxMessageLength-len(truncMsg))] + truncMsg
+	}
+
 	if foundNonCompliant {
 		return metav1.Condition{
 			Type:               compliantConditionType,
 			Status:             metav1.ConditionFalse,
 			LastTransitionTime: metav1.Now(),
 			Reason:             "NonCompliant",
-			Message:            "NonCompliant; " + strings.Join(messages, ", "),
+			Message:            message,
 		}
 	}
 
@@ -317,7 +330,7 @@ func calculateComplianceCondition(policy *policyv1beta1.OperatorPolicy) metav1.C
 		Status:             metav1.ConditionTrue,
 		LastTransitionTime: metav1.Now(),
 		Reason:             "Compliant",
-		Message:            "Compliant; " + strings.Join(messages, ", "),
+		Message:            message,
 	}
 }
 
@@ -334,11 +347,11 @@ func (r *OperatorPolicyReconciler) emitComplianceEvent(
 	}
 
 	ownerRef := policy.OwnerReferences[0]
-	now := time.Now()
+	timestamp := complianceCondition.LastTransitionTime
 	event := &corev1.Event{
 		ObjectMeta: metav1.ObjectMeta{
 			// This event name matches the convention of recorders from client-go
-			Name:      fmt.Sprintf("%v.%x", ownerRef.Name, now.UnixNano()),
+			Name:      fmt.Sprintf("%v.%x", ownerRef.Name, timestamp.UnixNano()),
 			Namespace: policy.Namespace,
 		},
 		InvolvedObject: corev1.ObjectReference{
@@ -354,8 +367,8 @@ func (r *OperatorPolicyReconciler) emitComplianceEvent(
 			Component: ControllerName,
 			Host:      r.InstanceName,
 		},
-		FirstTimestamp: metav1.NewTime(now),
-		LastTimestamp:  metav1.NewTime(now),
+		FirstTimestamp: timestamp,
+		LastTimestamp:  timestamp,
 		Count:          1,
 		Type:           "Normal",
 		Action:         "ComplianceStateUpdate",

controllers/operatorpolicy_status.go

@@ -310,6 +310,21 @@ spec:
                 x-kubernetes-list-map-keys:
                 - type
                 x-kubernetes-list-type: map
+              history:
+                description: |-
+                  History is a list of the most recent compliance messages for this operator policy.
+                  The first entry is the most recent, and the list is limited to 10 entries.
+                items:
+                  description: HistoryEvent is a timestamped message representing
+                    the policy compliance state at that time.
+                  properties:
+                    lastTimestamp:
+                      format: date-time
+                      type: string
+                    message:
+                      type: string
+                  type: object
+                type: array
               observedGeneration:
                 description: ObservedGeneration is the latest generation observed
                   by the controller.

deploy/crds/kustomize_operatorpolicy/policy.open-cluster-management.io_operatorpolicies.yaml

@@ -312,6 +312,21 @@ spec:
                 x-kubernetes-list-map-keys:
                 - type
                 x-kubernetes-list-type: map
+              history:
+                description: |-
+                  History is a list of the most recent compliance messages for this operator policy.
+                  The first entry is the most recent, and the list is limited to 10 entries.
+                items:
+                  description: HistoryEvent is a timestamped message representing
+                    the policy compliance state at that time.
+                  properties:
+                    lastTimestamp:
+                      format: date-time
+                      type: string
+                    message:
+                      type: string
+                  type: object
+                type: array
               observedGeneration:
                 description: ObservedGeneration is the latest generation observed
                   by the controller.

deploy/crds/policy.open-cluster-management.io_operatorpolicies.yaml

@@ -15,7 +15,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
-	policyv1 "open-cluster-management.io/config-policy-controller/api/v1"
 	"open-cluster-management.io/config-policy-controller/test/utils"
 )
 
@@ -316,10 +315,10 @@ var _ = Describe("When standalone-hub-templates is enabled", Ordered, Label("hub
 
 			By("Checking the compliance message")
 			desiredMessage := "NonCompliant; failed to resolve the template"
-			Eventually(func() []policyv1.HistoryEvent {
-				return utils.GetHistoryEvents(clientManagedDynamic, gvrConfigPolicy,
-					policyName, testNamespace, desiredMessage)
-			}, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty())
+			Eventually(func() []string {
+				return utils.GetHistoryMessages(clientManagedDynamic, gvrOperatorPolicy,
+					policyName, testNamespace, "")
+			}, defaultTimeoutSeconds, 1).Should(ContainElement(ContainSubstring(desiredMessage)))
 			Eventually(func() []v1.Event {
 				return utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName,
 					fmt.Sprintf("policy: %v/%v", testNamespace, policyName), desiredMessage, defaultTimeoutSeconds)
@@ -331,10 +330,10 @@ var _ = Describe("When standalone-hub-templates is enabled", Ordered, Label("hub
 
 			By("Checking the compliance message")
 			desiredMessage := `NonCompliant; the operator namespace \('hello'\) does not exist,`
-			Eventually(func() []policyv1.HistoryEvent {
-				return utils.GetHistoryEvents(clientManagedDynamic, gvrConfigPolicy,
-					policyName, testNamespace, desiredMessage)
-			}, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty())
+			Eventually(func() []string {
+				return utils.GetHistoryMessages(clientManagedDynamic, gvrOperatorPolicy,
+					policyName, testNamespace, "")
+			}, defaultTimeoutSeconds, 1).Should(ContainElement(MatchRegexp(desiredMessage)))
 			Eventually(func() []v1.Event {
 				return utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName,
 					fmt.Sprintf("policy: %v/%v", testNamespace, policyName), desiredMessage, defaultTimeoutSeconds)
@@ -349,10 +348,10 @@ var _ = Describe("When standalone-hub-templates is enabled", Ordered, Label("hub
 
 			By("Checking the compliance message")
 			desiredMessage := `NonCompliant; the operator namespace \('changed'\) does not exist,`
-			Eventually(func() []policyv1.HistoryEvent {
-				return utils.GetHistoryEvents(clientManagedDynamic, gvrConfigPolicy,
-					policyName, testNamespace, desiredMessage)
-			}, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty())
+			Eventually(func() []string {
+				return utils.GetHistoryMessages(clientManagedDynamic, gvrOperatorPolicy,
+					policyName, testNamespace, "")
+			}, defaultTimeoutSeconds, 1).Should(ContainElement(MatchRegexp(desiredMessage)))
 			Eventually(func() []v1.Event {
 				return utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName,
 					fmt.Sprintf("policy: %v/%v", testNamespace, policyName), desiredMessage, defaultTimeoutSeconds)
@@ -365,10 +364,10 @@ var _ = Describe("When standalone-hub-templates is enabled", Ordered, Label("hub
 
 			By("Checking the compliance message")
 			desiredMessage := `NonCompliant; the namespace '{{hub fromConfigMap`
-			Eventually(func() []policyv1.HistoryEvent {
-				return utils.GetHistoryEvents(clientManagedDynamic, gvrConfigPolicy,
-					policyName, testNamespace, desiredMessage)
-			}, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty())
+			Eventually(func() []string {
+				return utils.GetHistoryMessages(clientManagedDynamic, gvrOperatorPolicy,
+					policyName, testNamespace, "")
+			}, defaultTimeoutSeconds, 1).Should(ContainElement(ContainSubstring(desiredMessage)))
 			Eventually(func() []v1.Event {
 				return utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName,
 					fmt.Sprintf("policy: %v/%v", testNamespace, policyName), desiredMessage, defaultTimeoutSeconds)