Add alert + kpi for knowledge status conditions

Author: PhilippMatthes

helm/bundles/cortex-cinder/alerts/cinder.alerts.yaml

@@ -159,7 +159,7 @@ groups:
         service will have a less recent view of the datacenter.
 
   - alert: CortexCinderDatasourceUnready
-    expr: cortex_datasource_state{operator="cortex-cinder",state=~"waiting|error|unknown"} != 0
+    expr: cortex_datasource_state{operator="cortex-cinder",state!="ready"} != 0
     for: 60m
     labels:
       context: datasources
@@ -173,3 +173,19 @@ groups:
         This may indicate issues with the datasource
         connectivity or configuration. It is recommended to investigate the
         datasource status and logs for more details.
+
+  - alert: CortexCinderKnowledgeUnready
+    expr: cortex_knowledge_state{operator="cortex-cinder",state!="ready"} != 0
+    for: 60m
+    labels:
+      context: knowledge
+      dashboard: cortex/cortex
+      service: cortex
+      severity: warning
+      support_group: workload-management
+    annotations:
+      summary: "Knowledge `{{$labels.knowledge}}` is in `{{$labels.state}}` state"
+      description: >
+        This may indicate issues with the knowledge
+        configuration. It is recommended to investigate the
+        knowledge status and logs for more details.

helm/bundles/cortex-cinder/templates/kpis.yaml

@@ -10,3 +10,15 @@ spec:
     datasourceOperator: cortex-cinder
   description: |
     This KPI tracks the state of datasource resources managed by cortex.
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: KPI
+metadata:
+  name: cortex-cinder-knowledge-state-kpi
+spec:
+  operator: cortex-cinder
+  impl: knowledge_state_kpi
+  opts:
+    knowledgeOperator: cortex-cinder
+  description: |
+    This KPI tracks the state of knowledge resources managed by cortex.

helm/bundles/cortex-manila/alerts/manila.alerts.yaml

@@ -159,7 +159,7 @@ groups:
         service will have a less recent view of the datacenter.
 
   - alert: CortexManilaDatasourceUnready
-    expr: cortex_datasource_state{operator="cortex-manila",state=~"waiting|error|unknown"} != 0
+    expr: cortex_datasource_state{operator="cortex-manila",state!="ready"} != 0
     for: 60m
     labels:
       context: datasources
@@ -173,3 +173,19 @@ groups:
         This may indicate issues with the datasource
         connectivity or configuration. It is recommended to investigate the
         datasource status and logs for more details.
+
+  - alert: CortexManilaKnowledgeUnready
+    expr: cortex_knowledge_state{operator="cortex-manila",state!="ready"} != 0
+    for: 60m
+    labels:
+      context: knowledge
+      dashboard: cortex/cortex
+      service: cortex
+      severity: warning
+      support_group: workload-management
+    annotations:
+      summary: "Knowledge `{{$labels.knowledge}}` is in `{{$labels.state}}` state"
+      description: >
+        This may indicate issues with the knowledge
+        configuration. It is recommended to investigate the
+        knowledge status and logs for more details.

helm/bundles/cortex-manila/templates/kpis.yaml

@@ -23,3 +23,15 @@ spec:
     datasourceOperator: cortex-manila
   description: |
     This KPI tracks the state of datasource resources managed by cortex.
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: KPI
+metadata:
+  name: cortex-manila-knowledge-state-kpi
+spec:
+  operator: cortex-manila
+  impl: knowledge_state_kpi
+  opts:
+    knowledgeOperator: cortex-manila
+  description: |
+    This KPI tracks the state of knowledge resources managed by cortex.

helm/bundles/cortex-nova/alerts/nova.alerts.yaml

@@ -175,7 +175,7 @@ groups:
         service will have a less recent view of the datacenter.
 
   - alert: CortexNovaDatasourceUnready
-    expr: cortex_datasource_state{operator="cortex-nova",state=~"waiting|error|unknown"} != 0
+    expr: cortex_datasource_state{operator="cortex-nova",state!="ready"} != 0
     for: 60m
     labels:
       context: datasources
@@ -189,3 +189,19 @@ groups:
         This may indicate issues with the datasource
         connectivity or configuration. It is recommended to investigate the
         datasource status and logs for more details.
+
+  - alert: CortexNovaKnowledgeUnready
+    expr: cortex_knowledge_state{operator="cortex-nova",state!="ready"} != 0
+    for: 60m
+    labels:
+      context: knowledge
+      dashboard: cortex/cortex
+      service: cortex
+      severity: warning
+      support_group: workload-management
+    annotations:
+      summary: "Knowledge `{{$labels.knowledge}}` is in `{{$labels.state}}` state"
+      description: >
+        This may indicate issues with the knowledge
+        configuration. It is recommended to investigate the
+        knowledge status and logs for more details.

helm/bundles/cortex-nova/templates/kpis.yaml

@@ -147,3 +147,15 @@ spec:
     datasourceOperator: cortex-nova
   description: |
     This KPI tracks the state of datasource resources managed by cortex.
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: KPI
+metadata:
+  name: cortex-nova-knowledge-state-kpi
+spec:
+  operator: cortex-nova
+  impl: knowledge_state_kpi
+  opts:
+    knowledgeOperator: cortex-nova
+  description: |
+    This KPI tracks the state of knowledge resources managed by cortex.

internal/knowledge/kpis/plugins/deployment/knowledge_state.go

@@ -0,0 +1,81 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package deployment
+
+import (
+	"context"
+
+	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
+	"github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins"
+	"github.com/cobaltcore-dev/cortex/pkg/conf"
+	"github.com/cobaltcore-dev/cortex/pkg/db"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type KnowledgeStateKPIOpts struct {
+	// The operator to filter knowledges by.
+	KnowledgeOperator string `yaml:"knowledgeOperator"`
+}
+
+// KPI observing the state of knowledge resources managed by cortex.
+type KnowledgeStateKPI struct {
+	// Common base for all KPIs that provides standard functionality.
+	plugins.BaseKPI[KnowledgeStateKPIOpts]
+
+	// Prometheus descriptor for the knowledge state metric.
+	counter *prometheus.Desc
+}
+
+func (KnowledgeStateKPI) GetName() string { return "knowledge_state_kpi" }
+
+// Initialize the KPI.
+func (k *KnowledgeStateKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error {
+	if err := k.BaseKPI.Init(db, client, opts); err != nil {
+		return err
+	}
+	k.counter = prometheus.NewDesc(
+		"cortex_knowledge_state",
+		"State of cortex managed knowledges",
+		[]string{"operator", "knowledge", "state"},
+		nil,
+	)
+	return nil
+}
+
+// Conform to the prometheus collector interface by providing the descriptor.
+func (k *KnowledgeStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter }
+
+// Collect the knowledge state metrics.
+func (k *KnowledgeStateKPI) Collect(ch chan<- prometheus.Metric) {
+	// Get all knowledges with the specified knowledge operator.
+	knowledgeList := &v1alpha1.KnowledgeList{}
+	if err := k.Client.List(context.Background(), knowledgeList); err != nil {
+		return
+	}
+	var knowledges []v1alpha1.Knowledge
+	for _, ds := range knowledgeList.Items {
+		if ds.Spec.Operator != k.Options.KnowledgeOperator {
+			continue
+		}
+		knowledges = append(knowledges, ds)
+	}
+	// For each knowledge, emit a metric with its state.
+	for _, ds := range knowledges {
+		var state string
+		switch {
+		case meta.IsStatusConditionTrue(ds.Status.Conditions, v1alpha1.KnowledgeConditionError):
+			state = "error"
+		case ds.Status.IsReady():
+			state = "ready"
+		default:
+			state = "unknown"
+		}
+		ch <- prometheus.MustNewConstMetric(
+			k.counter, prometheus.GaugeValue, 1,
+			k.Options.KnowledgeOperator, ds.Name, state,
+		)
+	}
+}