Deploy CSI-Addons with volume condition reporter

When a Driver gets annotated with

    addons.csi.ceph.io/volume-condition: true

the CSI-Addons DaemonSet for that driver will get deployed with the
extra `--enable-volume-condition=true` commandline flag. This causes the
CSI-Addons sidecar to periodically check attached volumes for this
Driver and report the condition in the logs of the sidecar and as an
Event for the PersistentVolumeClaim.

Signed-off-by: Niels de Vos <ndevos@ibm.com>

Author: nixpanic

PendingReleaseNotes.md

@@ -4,4 +4,6 @@
 
 ## Features
 
+- CSI-Addons Volume Condition reporting can be enabled by annotating a Driver with `addons.csi.ceph.io/volume-condition: true`.
+
 ## NOTE

config/csi-rbac/cephfs_nodeplugin_cluster_role.yaml

@@ -19,3 +19,10 @@ rules:
 - apiGroups: [""]
   resources: ["serviceaccounts/token"]
   verbs: ["create"]
+# events and pv, pvc for volume condition reporter
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["list", "watch", "create", "update", "patch"]
+- apiGroups: [""]
+  resources: ["persistentvolumes", "persistentvolumeclaims"]
+  verbs: ["get"]

config/csi-rbac/cephfs_nodeplugin_role.yaml

@@ -0,0 +1,17 @@
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cephfs-nodeplugin-r
+rules:
+  - apiGroups: ["csiaddons.openshift.io"]
+    resources: ["csiaddonsnodes"]
+    verbs: ["get", "watch", "list", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get"]
+  - apiGroups: ["apps"]
+    resources: ["replicasets"]
+    verbs: ["get"]
+  - apiGroups: ["apps"]
+    resources: ["deployments/finalizers", "daemonsets/finalizers"]
+    verbs: ["update"]

config/csi-rbac/cephfs_nodeplugin_role_binding.yaml

@@ -0,0 +1,12 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cephfs-nodeplugin-rb
+subjects:
+  - kind: ServiceAccount
+    name: cephfs-nodeplugin-sa
+    namespace: system
+roleRef:
+  kind: Role
+  name: cephfs-nodeplugin-r
+  apiGroup: rbac.authorization.k8s.io

config/csi-rbac/rbd_nodeplugin_cluster_role.yaml

@@ -27,3 +27,10 @@ rules:
   - apiGroups: ["authentication.k8s.io"]
     resources: ["tokenreviews"]
     verbs: ["create"]
+# events and pvc for volume condition reporter
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get"]

internal/controller/driver_controller.go

@@ -26,6 +26,7 @@ import (
 	"reflect"
 	"regexp"
 	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/go-logr/logr"
@@ -72,7 +73,10 @@ const (
 const (
 	// Annotation name for ownerref information
 	ownerRefAnnotationKey = "csi.ceph.io/ownerref"
-	logRotateCmd          = `while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`
+	// Annotation to enable CSI-Addons volume condition reporter
+	driverCSIAddonsFeatureVolumeCondition = "addons.csi.ceph.io/volume-condition"
+
+	logRotateCmd = `while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`
 )
 
 // A regexp used to parse driver's prefix and type from the full name
@@ -236,10 +240,7 @@ func (r *driverReconcile) reconcile() error {
 		r.reconcileControllerPluginDeployment,
 		r.reconcileNodePluginDeamonSet,
 		r.reconcileLivenessService,
-	}
-
-	if r.isRbdDriver() {
-		reconcilers = append(reconcilers, r.reconcileNodePluginDeamonSetForCsiAddons)
+		r.reconcileNodePluginDeamonSetForCsiAddons,
 	}
 
 	// Concurrently reconcile different aspects of the clusters actual state to meet
@@ -981,7 +982,38 @@ func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
 
 	log := r.log.WithValues("csiAddonsDaemonSetName", daemonSet.Name)
 
-	if !ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
+	withCsiAddonsDaemonSet := false
+	withCsiAddonsVolumeCondition := false
+
+	if r.isRbdDriver() {
+		withCsiAddonsDaemonSet = ptr.Deref(r.driver.Spec.DeployCsiAddons, false)
+	}
+
+	// check if the driver wants CSI-Addons features
+	if feature := r.driver.GetAnnotations()[driverCSIAddonsFeatureVolumeCondition]; feature != "" {
+		enabled, err := strconv.ParseBool(feature)
+		if err != nil {
+			r.log.Error(
+				err,
+				"Unable to parse annotation on driver.csi.ceph.io",
+				"name",
+				client.ObjectKeyFromObject(&r.driver),
+				driverCSIAddonsFeatureVolumeCondition,
+				feature,
+			)
+			return err
+		}
+
+		withCsiAddonsVolumeCondition = enabled
+		// if the feature is enabled, enable the daemonset too
+		withCsiAddonsDaemonSet = withCsiAddonsDaemonSet || withCsiAddonsVolumeCondition
+	}
+
+	if r.isNfsDriver() {
+		withCsiAddonsDaemonSet = false
+	}
+
+	if !withCsiAddonsDaemonSet {
 		if err := r.Delete(r.ctx, daemonSet); client.IgnoreNotFound(err) != nil {
 			log.Error(err, "failed to delete csi addons daemonset")
 			return err
@@ -1062,6 +1094,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
 										utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
 										utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
 										utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
+										utils.If(withCsiAddonsVolumeCondition, utils.CsiAddonsVolumeConditionArg, ""),
 									},
 								),
 								Ports: []corev1.ContainerPort{
@@ -1080,6 +1113,12 @@ func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
 									if logRotationEnabled {
 										mounts = append(mounts, utils.LogsDirVolumeMount)
 									}
+									if withCsiAddonsVolumeCondition {
+										mounts = append(mounts,
+											utils.PluginMountDirVolumeMount(kubeletDirPath),
+											utils.PodsMountDirVolumeMount(kubeletDirPath),
+										)
+									}
 									return mounts
 								}),
 								Resources: ptr.Deref(
@@ -1124,6 +1163,14 @@ func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
 								utils.LogRotateDirVolumeName(r.driver.Name),
 							)
 						}
+
+						if withCsiAddonsVolumeCondition {
+							volumes = append(
+								volumes,
+								utils.PluginMountDirVolume(kubeletDirPath),
+								utils.PodsMountDirVolume(kubeletDirPath),
+							)
+						}
 						return volumes
 					}),
 				},

internal/utils/csi.go

@@ -400,6 +400,7 @@ var EnableVolumeGroupSnapshotsContainerArg = "--feature-gates=CSIVolumeGroupSnap
 var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true"
 var LogToStdErrContainerArg = "--logtostderr=false"
 var AlsoLogToStdErrContainerArg = "--alsologtostderr=true"
+var CsiAddonsVolumeConditionArg = "--enable-volume-condition=true"
 
 func LogVerbosityContainerArg(level int) string {
 	return fmt.Sprintf("--v=%d", Clamp(level, 0, 5))