fixes

Author: leandroberetta

internal/pkg/metrics/alerts/builder.go

@@ -36,50 +36,102 @@ type ruleBuilder struct {
 // BuildRules is the main entry point that builds both alerts and recording rules
 // based on each health rule's mode configuration
 func BuildRules(ctx context.Context, fc *flowslatest.FlowCollectorSpec) []monitoringv1.Rule {
-	log := log.FromContext(ctx)
 	rules := []monitoringv1.Rule{}
 
 	if fc.HasExperimentalAlertsHealth() {
 		healthRules := fc.GetHealthRules()
 		metrics := fc.GetIncludeList()
 
-		for _, healthRule := range healthRules {
-			if ok, _ := healthRule.IsAllowed(fc); !ok {
-				continue
+		// Build health rules and track system rules mode
+		addNoFlowsRecording, addLokiErrorRecording := buildHealthRules(ctx, fc, healthRules, metrics, &rules)
+
+		// Add system health rules based on mode
+		addSystemHealthRules(fc, addNoFlowsRecording, addLokiErrorRecording, &rules)
+	} else {
+		// If experimental alerts are not enabled, add default alert rules
+		addDefaultSystemAlerts(fc, &rules)
+	}
+
+	return rules
+}
+
+// buildHealthRules processes health rules and returns flags indicating if NoFlows/LokiError should be recording rules
+func buildHealthRules(ctx context.Context, fc *flowslatest.FlowCollectorSpec, healthRules []flowslatest.HealthRule, metrics []string, rules *[]monitoringv1.Rule) (bool, bool) {
+	log := log.FromContext(ctx)
+	addNoFlowsRecording := false
+	addLokiErrorRecording := false
+
+	for _, healthRule := range healthRules {
+		if ok, _ := healthRule.IsAllowed(fc); !ok {
+			continue
+		}
+
+		// Check if NoFlows or LokiError are configured as recording rules
+		if healthRule.Mode == flowslatest.HealthRuleModeRecordingRule {
+			if healthRule.Template == flowslatest.AlertNoFlows {
+				addNoFlowsRecording = true
+			}
+			if healthRule.Template == flowslatest.AlertLokiError {
+				addLokiErrorRecording = true
 			}
+		}
 
-			for _, variant := range healthRule.Variants {
-				var r []monitoringv1.Rule
-				var err error
+		for _, variant := range healthRule.Variants {
+			var r []monitoringv1.Rule
+			var err error
 
-				// Decide whether to build alert or recording rule based on mode
-				if healthRule.Mode == flowslatest.HealthRuleModeRecordingRule {
-					r, err = convertToRecordingRules(healthRule.Template, &variant, metrics)
-				} else {
-					// Default to alert mode
-					r, err = convertToRules(healthRule.Template, &variant, metrics)
-				}
+			// Decide whether to build alert or recording rule based on mode
+			if healthRule.Mode == flowslatest.HealthRuleModeRecordingRule {
+				r, err = convertToRecordingRules(healthRule.Template, &variant, metrics)
+			} else {
+				// Default to alert mode
+				r, err = convertToRules(healthRule.Template, &variant, metrics)
+			}
 
-				if err != nil {
-					log.Error(err, "unable to configure a health rule", "template", healthRule.Template, "mode", healthRule.Mode)
-				} else if len(r) > 0 {
-					rules = append(rules, r...)
-				}
+			if err != nil {
+				log.Error(err, "unable to configure a health rule", "template", healthRule.Template, "mode", healthRule.Mode)
+			} else if len(r) > 0 {
+				*rules = append(*rules, r...)
 			}
 		}
 	}
 
-	// Add system health rules (NoFlows and LokiError)
+	return addNoFlowsRecording, addLokiErrorRecording
+}
+
+// addSystemHealthRules adds NoFlows and LokiError rules based on configured mode
+func addSystemHealthRules(fc *flowslatest.FlowCollectorSpec, addNoFlowsRecording, addLokiErrorRecording bool, rules *[]monitoringv1.Rule) {
+	// Add system health recording rules (NoFlows and LokiError) if configured
+	if addNoFlowsRecording && !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertNoFlows) {
+		r := RecordingNoFlows()
+		*rules = append(*rules, *r)
+	}
+	if addLokiErrorRecording && !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertLokiError) {
+		r := RecordingLokiError()
+		*rules = append(*rules, *r)
+	}
+
+	// Add system health alert rules (NoFlows and LokiError) - only if not configured as recording rules
+	if !addNoFlowsRecording && !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertNoFlows) {
+		r := alertNoFlows()
+		*rules = append(*rules, *r)
+	}
+	if !addLokiErrorRecording && !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertLokiError) {
+		r := alertLokiError()
+		*rules = append(*rules, *r)
+	}
+}
+
+// addDefaultSystemAlerts adds default alert rules when experimental alerts are not enabled
+func addDefaultSystemAlerts(fc *flowslatest.FlowCollectorSpec, rules *[]monitoringv1.Rule) {
 	if !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertNoFlows) {
 		r := alertNoFlows()
-		rules = append(rules, *r)
+		*rules = append(*rules, *r)
 	}
 	if !slices.Contains(fc.Processor.Metrics.DisableAlerts, flowslatest.AlertLokiError) {
 		r := alertLokiError()
-		rules = append(rules, *r)
+		*rules = append(*rules, *r)
 	}
-
-	return rules
 }
 
 func convertToRules(template flowslatest.AlertTemplate, alert *flowslatest.HealthRuleVariant, enabledMetrics []string) ([]monitoringv1.Rule, error) {

internal/pkg/metrics/alerts/recording.go

@@ -215,7 +215,7 @@ func (rb *ruleBuilder) latencyTrendRecording() (*monitoringv1.Rule, error) {
 	}, nil
 }
 
-func recordingNoFlows() *monitoringv1.Rule {
+func RecordingNoFlows() *monitoringv1.Rule {
 	return &monitoringv1.Rule{
 		Record: "netobserv:health:no_flows:rate1m",
 		Expr:   intstr.FromString("sum(rate(netobserv_ingest_flows_processed[1m]))"),
@@ -226,7 +226,7 @@ func recordingNoFlows() *monitoringv1.Rule {
 	}
 }
 
-func recordingLokiError() *monitoringv1.Rule {
+func RecordingLokiError() *monitoringv1.Rule {
 	return &monitoringv1.Rule{
 		Record: "netobserv:health:loki_errors:rate1m",
 		Expr:   intstr.FromString("sum(rate(netobserv_loki_dropped_entries_total[1m]))"),

internal/pkg/metrics/alerts/recording_test.go

@@ -207,3 +207,59 @@ func TestRecordingRuleLabels(t *testing.T) {
 	assert.Equal(t, "Workload", labels["health_groupby"])
 	assert.Equal(t, "Dst", labels["health_side"])
 }
+
+func TestBuildRules_SystemRecordingRules(t *testing.T) {
+	ctx := context.Background()
+
+	// Create health rules with NoFlows and LokiError as recording rules
+	healthRules := []flowslatest.HealthRule{
+		{
+			Template: flowslatest.AlertNoFlows,
+			Mode:     flowslatest.HealthRuleModeRecordingRule,
+			Variants: []flowslatest.HealthRuleVariant{{}},
+		},
+		{
+			Template: flowslatest.AlertLokiError,
+			Mode:     flowslatest.HealthRuleModeRecordingRule,
+			Variants: []flowslatest.HealthRuleVariant{{}},
+		},
+	}
+
+	spec := &flowslatest.FlowCollectorSpec{
+		Processor: flowslatest.FlowCollectorFLP{
+			Metrics: flowslatest.FLPMetrics{
+				HealthRules: &healthRules,
+			},
+			Advanced: &flowslatest.AdvancedProcessorConfig{
+				Env: map[string]string{
+					"EXPERIMENTAL_ALERTS_HEALTH": "true",
+				},
+			},
+		},
+	}
+
+	rules := BuildRules(ctx, spec)
+
+	// Should have recording rules for NoFlows and LokiError
+	var noFlowsRecording, lokiErrorRecording bool
+	for _, rule := range rules {
+		if rule.Record == "netobserv:health:no_flows:rate1m" {
+			noFlowsRecording = true
+			assert.Equal(t, "health", rule.Labels["netobserv"])
+			assert.Equal(t, "NetObservNoFlows", rule.Labels["health_template"])
+			assert.Contains(t, rule.Expr.StrVal, "netobserv_ingest_flows_processed")
+		}
+		if rule.Record == "netobserv:health:loki_errors:rate1m" {
+			lokiErrorRecording = true
+			assert.Equal(t, "health", rule.Labels["netobserv"])
+			assert.Equal(t, "NetObservLokiError", rule.Labels["health_template"])
+			assert.Contains(t, rule.Expr.StrVal, "netobserv_loki_dropped_entries_total")
+		}
+		// Should not have alert versions
+		assert.NotEqual(t, "NetObservNoFlows", rule.Alert)
+		assert.NotEqual(t, "NetObservLokiError", rule.Alert)
+	}
+
+	assert.True(t, noFlowsRecording, "should have NoFlows recording rule")
+	assert.True(t, lokiErrorRecording, "should have LokiError recording rule")
+}