Merge pull request #573 from openstack-k8s-operators/OSPRH-11017/add-nad-metricstorages

openshift-merge-bot[bot] · web-flow · commit bf6aa818644b · 2025-02-04T09:50:11.000Z
Allow attaching MetricStorage pod to additional networks
diff --git a/api/bases/telemetry.openstack.org_ceilometers.yaml b/api/bases/telemetry.openstack.org_ceilometers.yaml
@@ -185,6 +185,7 @@ spec:
                 items:
                   type: string
                 type: array
+                x-kubernetes-list-type: atomic
               nodeSelector:
                 additionalProperties:
                   type: string
diff --git a/api/bases/telemetry.openstack.org_metricstorages.yaml b/api/bases/telemetry.openstack.org_metricstorages.yaml
@@ -15,6 +15,10 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
+    - description: NetworkAttachments
+      jsonPath: .status.networkAttachments
+      name: NetworkAttachments
+      type: string
     - description: Status
       jsonPath: .status.conditions[0].status
       name: Status
@@ -1299,6 +1303,13 @@ spec:
                         type: string
                     type: object
                 type: object
+              networkAttachments:
+                description: NetworkAttachments is a list of NetworkAttachment resource
+                  names to expose the services to the given network
+                items:
+                  type: string
+                type: array
+                x-kubernetes-list-type: atomic
               prometheusTls:
                 description: TLS - Parameters related to the TLS
                 properties:
@@ -1357,6 +1368,11 @@ spec:
                   - type
                   type: object
                 type: array
+              networkAttachments:
+                additionalProperties:
+                  type: string
+                description: NetworkAttachments status of the Prometheus pods
+                type: object
               observedGeneration:
                 description: |-
                   ObservedGeneration - the most recent generation observed for this
diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml
@@ -481,6 +481,7 @@ spec:
                     items:
                       type: string
                     type: array
+                    x-kubernetes-list-type: atomic
                   nodeSelector:
                     additionalProperties:
                       type: string
@@ -1868,6 +1869,13 @@ spec:
                             type: string
                         type: object
                     type: object
+                  networkAttachments:
+                    description: NetworkAttachments is a list of NetworkAttachment
+                      resource names to expose the services to the given network
+                    items:
+                      type: string
+                    type: array
+                    x-kubernetes-list-type: atomic
                   prometheusTls:
                     description: TLS - Parameters related to the TLS
                     properties:
diff --git a/api/v1beta1/ceilometer_types.go b/api/v1beta1/ceilometer_types.go
@@ -111,6 +111,7 @@ type CeilometerSpecCore struct {
 	DefaultConfigOverwrite map[string]string `json:"defaultConfigOverwrite,omitempty"`
 
 	// NetworkAttachmentDefinitions list of network attachment definitions the service pod gets attached to
+	// +listType=atomic
 	NetworkAttachmentDefinitions []string `json:"networkAttachmentDefinitions,omitempty"`
 
 	// Whether kube-state-metrics should be deployed
diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go
@@ -181,6 +181,9 @@ const (
 	// PrometheusUnableToRemoveTLSMessage
 	PrometheusUnableToRemoveTLSMessage = "Error occured when trying to remove TLS config: %s"
 
+	// PrometheusUnableToRemoveNADMessage
+	PrometheusUnableToRemoveNADMessage = "Error occured when trying to remove NAD config: %s"
+
 	//
 	// LoggingReady condition messages
 	//
diff --git a/api/v1beta1/metricstorage_types.go b/api/v1beta1/metricstorage_types.go
@@ -90,6 +90,11 @@ type MetricStorageSpec struct {
 	// +kubebuilder:default=ctlplane
 	DataplaneNetwork *infranetworkv1.NetNameStr `json:"dataplaneNetwork"`
 
+	// NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network
+	// +kubebuilder:validation:Optional
+	// +listType=atomic
+	NetworkAttachments []string `json:"networkAttachments,omitempty"`
+
 	// MonitoringStack allows to define a metric storage with
 	// options supported by Red Hat
 	// +kubebuilder:validation:Optional
@@ -127,10 +132,13 @@ type MetricStorageStatus struct {
 	// then the controller has not processed the latest changes injected by
 	// the openstack-operator in the top-level CR (e.g. the ContainerImage)
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+	// NetworkAttachments status of the Prometheus pods
+	NetworkAttachments map[string]string `json:"networkAttachments,omitempty"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.networkAttachments",description="NetworkAttachments"
 //+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status"
 //+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message"
 
diff --git a/api/v1beta1/metricstorage_webhook.go b/api/v1beta1/metricstorage_webhook.go
@@ -59,6 +59,7 @@ func (spec *MetricStorageSpec) Default() {
 	if *spec.DataplaneNetwork == "" {
 		*spec.DataplaneNetwork = "ctlplane"
 	}
+
 	if spec.MonitoringStack != nil {
 		spec.MonitoringStack.Default()
 	}
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml
@@ -185,6 +185,7 @@ spec:
                 items:
                   type: string
                 type: array
+                x-kubernetes-list-type: atomic
               nodeSelector:
                 additionalProperties:
                   type: string
diff --git a/config/crd/bases/telemetry.openstack.org_metricstorages.yaml b/config/crd/bases/telemetry.openstack.org_metricstorages.yaml
@@ -15,6 +15,10 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
+    - description: NetworkAttachments
+      jsonPath: .status.networkAttachments
+      name: NetworkAttachments
+      type: string
     - description: Status
       jsonPath: .status.conditions[0].status
       name: Status
@@ -1299,6 +1303,13 @@ spec:
                         type: string
                     type: object
                 type: object
+              networkAttachments:
+                description: NetworkAttachments is a list of NetworkAttachment resource
+                  names to expose the services to the given network
+                items:
+                  type: string
+                type: array
+                x-kubernetes-list-type: atomic
               prometheusTls:
                 description: TLS - Parameters related to the TLS
                 properties:
@@ -1357,6 +1368,11 @@ spec:
                   - type
                   type: object
                 type: array
+              networkAttachments:
+                additionalProperties:
+                  type: string
+                description: NetworkAttachments status of the Prometheus pods
+                type: object
               observedGeneration:
                 description: |-
                   ObservedGeneration - the most recent generation observed for this
diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml
@@ -481,6 +481,7 @@ spec:
                     items:
                       type: string
                     type: array
+                    x-kubernetes-list-type: atomic
                   nodeSelector:
                     additionalProperties:
                       type: string
@@ -1868,6 +1869,13 @@ spec:
                             type: string
                         type: object
                     type: object
+                  networkAttachments:
+                    description: NetworkAttachments is a list of NetworkAttachment
+                      resource names to expose the services to the given network
+                    items:
+                      type: string
+                    type: array
+                    x-kubernetes-list-type: atomic
                   prometheusTls:
                     description: TLS - Parameters related to the TLS
                     properties:
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -102,6 +102,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - keystone.openstack.org
   resources:
diff --git a/controllers/metricstorage_controller.go b/controllers/metricstorage_controller.go
@@ -25,6 +25,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
@@ -55,9 +56,11 @@ import (
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
 	object "github.com/openstack-k8s-operators/lib-common/modules/common/object"
 	tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls"
 
+	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
 	telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1"
 	availability "github.com/openstack-k8s-operators/telemetry-operator/pkg/availability"
@@ -126,6 +129,7 @@ func (r *MetricStorageReconciler) GetLogger(ctx context.Context) logr.Logger {
 //+kubebuilder:rbac:groups=rabbitmq.com,resources=rabbitmqclusters,verbs=get;list;watch
 //+kubebuilder:rbac:groups=observability.openshift.io,resources=uiplugins,verbs=get;list;watch;create;patch
 //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
 
 // Reconcile reconciles MetricStorage
 func (r *MetricStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) {
@@ -196,6 +200,7 @@ func (r *MetricStorageReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		condition.UnknownCondition(telemetryv1.DashboardDefinitionReadyCondition, condition.InitReason, telemetryv1.DashboardDefinitionReadyInitMessage),
 		condition.UnknownCondition(telemetryv1.PrometheusReadyCondition, condition.InitReason, telemetryv1.PrometheusReadyInitMessage),
 		condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage),
+		condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage),
 	)
 
 	instance.Status.Conditions.Init(&cl)
@@ -510,6 +515,100 @@ func (r *MetricStorageReconciler) reconcileNormal(
 	// all cert input checks out so report InputReady
 	instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage)
 
+	//
+	// NAD
+	//
+
+	// Get networks to attach to the Prometheus pod
+	nadList := []networkv1.NetworkAttachmentDefinition{}
+
+	for _, netAtt := range instance.Spec.NetworkAttachments {
+		nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace)
+		if err != nil {
+			if k8s_errors.IsNotFound(err) {
+				instance.Status.Conditions.Set(condition.FalseCondition(
+					condition.NetworkAttachmentsReadyCondition,
+					condition.RequestedReason,
+					condition.SeverityInfo,
+					condition.NetworkAttachmentsReadyWaitingMessage,
+					netAtt))
+				return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("network-attachment-definition %s not found", netAtt)
+			}
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				condition.NetworkAttachmentsReadyCondition,
+				condition.ErrorReason,
+				condition.SeverityWarning,
+				condition.NetworkAttachmentsReadyErrorMessage,
+				err.Error()))
+			return ctrl.Result{}, err
+		}
+		if nad != nil {
+			nadList = append(nadList, *nad)
+		}
+	}
+
+	networkAnnotations, err := nad.EnsureNetworksAnnotation(nadList)
+
+	if err != nil {
+		err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err)
+		instance.Status.Conditions.MarkFalse(
+			condition.NetworkAttachmentsReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.NetworkAttachmentsReadyErrorMessage,
+			err)
+		return ctrl.Result{}, err
+	}
+
+	// Set NAD annotation to the Prometheus pod
+	if len(instance.Spec.NetworkAttachments) != 0 {
+		// Patch Prometheus to add the NAD annotation
+		prometheusWatchFn := func(_ context.Context, o client.Object) []reconcile.Request {
+			name := client.ObjectKey{
+				Namespace: o.GetNamespace(),
+				Name:      o.GetName(),
+			}
+			return []reconcile.Request{{NamespacedName: name}}
+		}
+		err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn))
+		if err != nil {
+			instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition,
+				condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"),
+				condition.SeverityError,
+				telemetryv1.PrometheusUnableToWatchMessage, err)
+			Log.Info("Can't watch Prometheus resource. The Cluster Observability Operator probably isn't installed")
+			return ctrl.Result{RequeueAfter: telemetryv1.PauseBetweenWatchAttempts}, nil
+		}
+		prometheusNADPatch := metricstorage.PrometheusNAD(instance, networkAnnotations)
+		err = r.Client.Patch(context.Background(), &prometheusNADPatch, client.Apply, client.FieldOwner("telemetry-operator"))
+		if err != nil {
+			Log.Error(err, "Can't patch Prometheus resource")
+			return ctrl.Result{}, err
+		}
+		instance.Status.NetworkAttachments = networkAnnotations
+	} else if len(instance.Status.NetworkAttachments) != 0 {
+		// Delete the prometheus CR, so it can be automatically restored without the NAD patch
+		prometheus := monv1.Prometheus{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: instance.Namespace,
+				Name:      instance.Name,
+			},
+		}
+		err = r.Client.Delete(context.Background(), &prometheus)
+		if err != nil && !k8s_errors.IsNotFound(err) {
+			instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition,
+				condition.Reason("Can't delete old Prometheus CR to remove NAD configuration"),
+				condition.SeverityError,
+				telemetryv1.PrometheusUnableToRemoveNADMessage, err)
+			Log.Error(err, "Can't delete old Prometheus CR to remove NAD configuration")
+			return ctrl.Result{}, err
+		}
+		instance.Status.NetworkAttachments = map[string]string{}
+	}
+
+	// when job passed, mark NetworkAttachmentsReadyCondition ready
+	instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage)
+
 	if instance.Status.Conditions.AllSubConditionIsTrue() {
 		instance.Status.Conditions.MarkTrue(
 			condition.ReadyCondition, condition.ReadyMessage)
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ replace github.com/openstack-k8s-operators/telemetry-operator/api => ./api
 
 require (
 	github.com/go-logr/logr v1.4.2
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5
 	github.com/onsi/ginkgo/v2 v2.20.1
 	github.com/onsi/gomega v1.34.1
 	github.com/openstack-k8s-operators/heat-operator/api v0.5.1-0.20250129204831-17deaa376556
diff --git a/go.sum b/go.sum
@@ -55,6 +55,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5 h1:CELpSMPSyicFBaVsxROmfrWlu9yr3Dduk+y7vGrIsx8=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.5/go.mod h1:CM7HAH5PNuIsqjMN0fGc1ydM74Uj+0VZFhob620nklw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
diff --git a/main.go b/main.go
@@ -39,6 +39,7 @@ import (
 
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
+	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1"
 	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
 	infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
@@ -76,6 +77,7 @@ func init() {
 	utilruntime.Must(heatv1.AddToScheme(scheme))
 	utilruntime.Must(infranetworkv1.AddToScheme(scheme))
 	utilruntime.Must(rabbitmqclusterv1.AddToScheme(scheme))
+	utilruntime.Must(networkv1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 
diff --git a/pkg/metricstorage/prometheus_nad_patch.go b/pkg/metricstorage/prometheus_nad_patch.go
diff --git a/tests/kuttl/suites/default/tests/01-assert.yaml b/tests/kuttl/suites/default/tests/01-assert.yaml
diff --git a/tests/kuttl/suites/metricstorage/tests/06-deploy-edpm-resources.yaml b/tests/kuttl/suites/metricstorage/tests/06-deploy-edpm-resources.yaml

Original file line number	Diff line number	Diff line change
`@@ -59,6 +59,7 @@ func (spec *MetricStorageSpec) Default() {`
`59`	`59`	`if *spec.DataplaneNetwork == "" {`
`60`	`60`	`*spec.DataplaneNetwork = "ctlplane"`
`61`	`61`	`}`
	`62`	`+`
`62`	`63`	`if spec.MonitoringStack != nil {`
`63`	`64`	`spec.MonitoringStack.Default()`
`64`	`65`	`}`