feat: populate SizeBytes in snapshots and add CSI maturity features

fenio · fenio · commit 062f111f9949 · 2026-02-23T19:42:11.000+01:00
Populate SizeBytes in all CreateSnapshot and ListSnapshots responses
using the source volume's capacity_bytes property, so VolumeSnapshot
objects report restoreSize correctly.

Add CSI capabilities and infrastructure that mature drivers implement:
- Advertise VolumeExpansion ONLINE plugin capability
- Advertise VOLUME_CONDITION controller capability for health monitoring
- Advertise SINGLE_NODE_MULTI_WRITER on controller and node
- Enable podInfoOnMount and storageCapacity in CSIDriver manifest
- Add livenessprobe sidecar to controller and node pods
- Enable --enable-capacity on csi-provisioner for capacity tracking
- Map capacity-related TrueNAS errors to ResourceExhausted gRPC code
diff --git a/charts/tns-csi-driver/templates/controller.yaml b/charts/tns-csi-driver/templates/controller.yaml
@@ -71,12 +71,23 @@ spec:
             - name: DEBUG_CSI
               value: "true"
             {{- end }}
-          {{- if .Values.controller.metrics.enabled }}
           ports:
+            {{- if .Values.controller.metrics.enabled }}
             - name: metrics
               containerPort: {{ .Values.controller.metrics.port }}
               protocol: TCP
-          {{- end }}
+            {{- end }}
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
@@ -97,9 +108,19 @@ spec:
             - "--leader-election"
             - "--default-fstype=nfs"
             - "--extra-create-metadata"
+            - "--enable-capacity"
+            - "--capacity-ownerref-level=2"
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
@@ -171,6 +192,22 @@ spec:
             {{- toYaml .Values.sidecars.snapshotter.resources | nindent 12 }}
         {{- end }}
 
+        # CSI Liveness Probe sidecar
+        - name: liveness-probe
+          image: "{{ .Values.sidecars.livenessprobe.image.repository }}:{{ .Values.sidecars.livenessprobe.image.tag }}"
+          imagePullPolicy: {{ .Values.sidecars.livenessprobe.image.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--health-port=9808"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          resources:
+            {{- toYaml .Values.sidecars.livenessprobe.resources | nindent 12 }}
+
       volumes:
         - name: socket-dir
           emptyDir: {}
diff --git a/charts/tns-csi-driver/templates/csidriver.yaml b/charts/tns-csi-driver/templates/csidriver.yaml
@@ -10,7 +10,8 @@ metadata:
   {{- end }}
 spec:
   attachRequired: false
-  podInfoOnMount: false
+  podInfoOnMount: true
+  storageCapacity: true
   fsGroupPolicy: File
   volumeLifecycleModes:
     - Persistent
diff --git a/charts/tns-csi-driver/templates/node.yaml b/charts/tns-csi-driver/templates/node.yaml
@@ -74,6 +74,18 @@ spec:
             - name: DEBUG_CSI
               value: "true"
             {{- end }}
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
           securityContext:
             privileged: true
             capabilities:
@@ -123,6 +135,22 @@ spec:
           resources:
             {{- toYaml .Values.sidecars.nodeDriverRegistrar.resources | nindent 12 }}
 
+        # CSI Liveness Probe sidecar
+        - name: liveness-probe
+          image: "{{ .Values.sidecars.livenessprobe.image.repository }}:{{ .Values.sidecars.livenessprobe.image.tag }}"
+          imagePullPolicy: {{ .Values.sidecars.livenessprobe.image.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--health-port=9808"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+          resources:
+            {{- toYaml .Values.sidecars.livenessprobe.resources | nindent 12 }}
+
       volumes:
         - name: plugin-dir
           hostPath:
diff --git a/charts/tns-csi-driver/values.yaml b/charts/tns-csi-driver/values.yaml
@@ -217,6 +217,19 @@ sidecars:
         cpu: 200m
         memory: 200Mi
 
+  livenessprobe:
+    image:
+      repository: registry.k8s.io/sig-storage/livenessprobe
+      tag: v2.14.0
+      pullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 10m
+        memory: 20Mi
+      limits:
+        cpu: 100m
+        memory: 100Mi
+
 # Storage class configuration
 # Each entry creates a Kubernetes StorageClass. You can have multiple classes
 # per protocol (e.g., two NFS classes with different reclaim policies).
diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go
@@ -59,6 +59,41 @@ var (
 	ErrDatasetNotFound = errors.New("dataset not found for share")
 )
 
+// capacityErrorSubstrings are error message patterns that indicate insufficient pool capacity.
+// TrueNAS returns these when a pool or dataset doesn't have enough free space.
+var capacityErrorSubstrings = []string{
+	"insufficient space",
+	"out of space",
+	"not enough space",
+	"no space left",
+	"ENOSPC",
+	"quota exceeded",
+}
+
+// isCapacityError checks if an error indicates a storage capacity issue.
+// Returns codes.ResourceExhausted status if it is, nil otherwise.
+func isCapacityError(err error) bool {
+	if err == nil {
+		return false
+	}
+	errStr := strings.ToLower(err.Error())
+	for _, substr := range capacityErrorSubstrings {
+		if strings.Contains(errStr, substr) {
+			return true
+		}
+	}
+	return false
+}
+
+// createVolumeError returns an appropriate gRPC status error for volume creation failures.
+// Maps capacity-related errors to ResourceExhausted per CSI spec.
+func createVolumeError(msg string, err error) error {
+	if isCapacityError(err) {
+		return status.Errorf(codes.ResourceExhausted, "%s: %v", msg, err)
+	}
+	return status.Errorf(codes.Internal, "%s: %v", msg, err)
+}
+
 // mountpointToDatasetID converts a ZFS mountpoint to a dataset ID.
 // ZFS datasets are mounted at /mnt/<dataset_name>, so we strip the /mnt/ prefix.
 // Example: /mnt/tank/csi/pvc-xxx -> tank/csi/pvc-xxx.
@@ -1509,6 +1544,20 @@ func (s *ControllerService) ControllerGetCapabilities(_ context.Context, _ *csi.
 					},
 				},
 			},
+			{
+				Type: &csi.ControllerServiceCapability_Rpc{
+					Rpc: &csi.ControllerServiceCapability_RPC{
+						Type: csi.ControllerServiceCapability_RPC_VOLUME_CONDITION,
+					},
+				},
+			},
+			{
+				Type: &csi.ControllerServiceCapability_Rpc{
+					Rpc: &csi.ControllerServiceCapability_RPC{
+						Type: csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
+					},
+				},
+			},
 		},
 	}, nil
 }
diff --git a/pkg/driver/controller_iscsi.go b/pkg/driver/controller_iscsi.go
@@ -424,7 +424,7 @@ func (s *ControllerService) getOrCreateZVOLForISCSI(ctx context.Context, params
 	zvol, err := s.apiClient.CreateZvol(ctx, createParams)
 	if err != nil {
 		timer.ObserveError()
-		return nil, status.Errorf(codes.Internal, "Failed to create ZVOL %s: %v", params.zvolName, err)
+		return nil, createVolumeError("Failed to create ZVOL "+params.zvolName, err)
 	}
 
 	klog.V(4).Infof("Created ZVOL: %s (ID: %s)", params.zvolName, zvol.ID)
diff --git a/pkg/driver/controller_iscsi_test.go b/pkg/driver/controller_iscsi_test.go
@@ -389,7 +389,7 @@ func TestCreateISCSIVolume(t *testing.T) {
 				}
 			},
 			wantErr:  true,
-			wantCode: codes.Internal,
+			wantCode: codes.ResourceExhausted,
 		},
 	}
 
diff --git a/pkg/driver/controller_nfs.go b/pkg/driver/controller_nfs.go
@@ -423,7 +423,7 @@ func (s *ControllerService) getOrCreateDataset(ctx context.Context, params *nfsV
 	dataset, err := s.apiClient.CreateDataset(ctx, createParams)
 	if err != nil {
 		timer.ObserveError()
-		return nil, status.Errorf(codes.Internal, "Failed to create dataset: %v", err)
+		return nil, createVolumeError("Failed to create dataset", err)
 	}
 
 	klog.V(4).Infof("Created dataset: %s with mountpoint: %s", dataset.Name, dataset.Mountpoint)
diff --git a/pkg/driver/controller_nvmeof.go b/pkg/driver/controller_nvmeof.go
@@ -620,7 +620,7 @@ func (s *ControllerService) getOrCreateZVOL(ctx context.Context, params *nvmeofV
 	zvol, err := s.apiClient.CreateZvol(ctx, createParams)
 	if err != nil {
 		timer.ObserveError()
-		return nil, status.Errorf(codes.Internal, "Failed to create ZVOL: %v", err)
+		return nil, createVolumeError("Failed to create ZVOL", err)
 	}
 
 	klog.V(4).Infof("Created ZVOL: %s (ID: %s)", zvol.Name, zvol.ID)
diff --git a/pkg/driver/controller_nvmeof_test.go b/pkg/driver/controller_nvmeof_test.go
@@ -313,7 +313,7 @@ func TestCreateNVMeOFVolume(t *testing.T) {
 				}
 			},
 			wantErr:  true,
-			wantCode: codes.Internal,
+			wantCode: codes.ResourceExhausted,
 		},
 		{
 			name: "subsystem creation failure with cleanup",
diff --git a/pkg/driver/controller_snapshot.go b/pkg/driver/controller_snapshot.go
@@ -290,16 +290,29 @@ func (s *ControllerService) CreateSnapshot(ctx context.Context, req *csi.CreateS
 		return nil, status.Errorf(codes.NotFound, "Source volume %s not found", sourceVolumeID)
 	}
 
+	// Query source volume capacity for SizeBytes in snapshot response
+	var sourceCapacityBytes int64
+	dataset, getErr := s.apiClient.GetDatasetWithProperties(ctx, datasetName)
+	if getErr == nil && dataset != nil {
+		if capProp, ok := dataset.UserProperties[tnsapi.PropertyCapacityBytes]; ok {
+			sourceCapacityBytes = tnsapi.StringToInt64(capProp.Value)
+		}
+		// Fallback: for ZVOLs, use volsize directly
+		if sourceCapacityBytes == 0 {
+			sourceCapacityBytes = getZvolCapacity(&dataset.Dataset)
+		}
+	}
+
 	// Route to appropriate snapshot creation method
 	if detached {
-		return s.createDetachedSnapshot(ctx, timer, snapshotName, sourceVolumeID, datasetName, protocol, pool, detachedParentDataset)
+		return s.createDetachedSnapshot(ctx, timer, snapshotName, sourceVolumeID, datasetName, protocol, pool, detachedParentDataset, sourceCapacityBytes)
 	}
 
-	return s.createRegularSnapshot(ctx, timer, snapshotName, sourceVolumeID, datasetName, protocol)
+	return s.createRegularSnapshot(ctx, timer, snapshotName, sourceVolumeID, datasetName, protocol, sourceCapacityBytes)
 }
 
 // createRegularSnapshot creates a traditional COW ZFS snapshot.
-func (s *ControllerService) createRegularSnapshot(ctx context.Context, timer *metrics.OperationTimer, snapshotName, sourceVolumeID, datasetName, protocol string) (*csi.CreateSnapshotResponse, error) {
+func (s *ControllerService) createRegularSnapshot(ctx context.Context, timer *metrics.OperationTimer, snapshotName, sourceVolumeID, datasetName, protocol string, sizeBytes int64) (*csi.CreateSnapshotResponse, error) {
 	klog.Infof("Creating regular snapshot %s for volume %s (dataset: %s, protocol: %s)",
 		snapshotName, sourceVolumeID, datasetName, protocol)
 
@@ -352,6 +365,7 @@ func (s *ControllerService) createRegularSnapshot(ctx context.Context, timer *me
 						SourceVolumeId: sourceVolumeID,
 						CreationTime:   timestamppb.New(time.Unix(createdAt, 0)),
 						ReadyToUse:     true, // ZFS snapshots are immediately available
+						SizeBytes:      sizeBytes,
 					},
 				}, nil
 			}
@@ -417,6 +431,7 @@ func (s *ControllerService) createRegularSnapshot(ctx context.Context, timer *me
 			SourceVolumeId: sourceVolumeID,
 			CreationTime:   timestamppb.New(time.Unix(createdAt, 0)),
 			ReadyToUse:     true, // ZFS snapshots are immediately available
+			SizeBytes:      sizeBytes,
 		},
 	}, nil
 }
diff --git a/pkg/driver/controller_snapshot_detached.go b/pkg/driver/controller_snapshot_detached.go
@@ -18,7 +18,7 @@ import (
 // createDetachedSnapshot creates a detached snapshot using zfs send/receive via TrueNAS replication API.
 // Detached snapshots are stored as full dataset copies, independent of the source volume.
 // They survive deletion of the source volume, making them suitable for backup/DR scenarios.
-func (s *ControllerService) createDetachedSnapshot(ctx context.Context, timer *metrics.OperationTimer, snapshotName, sourceVolumeID, sourceDataset, protocol, pool, detachedParentDataset string) (*csi.CreateSnapshotResponse, error) {
+func (s *ControllerService) createDetachedSnapshot(ctx context.Context, timer *metrics.OperationTimer, snapshotName, sourceVolumeID, sourceDataset, protocol, pool, detachedParentDataset string, sizeBytes int64) (*csi.CreateSnapshotResponse, error) {
 	// Determine the parent dataset for detached snapshots
 	if detachedParentDataset == "" {
 		if pool == "" {
@@ -85,6 +85,7 @@ func (s *ControllerService) createDetachedSnapshot(ctx context.Context, timer *m
 				SourceVolumeId: sourceVolumeID,
 				CreationTime:   timestamppb.New(time.Unix(createdAt, 0)),
 				ReadyToUse:     true,
+				SizeBytes:      sizeBytes,
 			},
 		}, nil
 	}
@@ -214,6 +215,7 @@ func (s *ControllerService) createDetachedSnapshot(ctx context.Context, timer *m
 			SourceVolumeId: sourceVolumeID,
 			CreationTime:   timestamppb.New(time.Unix(createdAt, 0)),
 			ReadyToUse:     true,
+			SizeBytes:      sizeBytes,
 		},
 	}, nil
 }
diff --git a/pkg/driver/controller_snapshot_list.go b/pkg/driver/controller_snapshot_list.go
diff --git a/pkg/driver/controller_snapshot_test.go b/pkg/driver/controller_snapshot_test.go
diff --git a/pkg/driver/identity.go b/pkg/driver/identity.go
diff --git a/pkg/driver/node.go b/pkg/driver/node.go

Original file line number	Diff line number	Diff line change
`@@ -424,7 +424,7 @@ func (s *ControllerService) getOrCreateZVOLForISCSI(ctx context.Context, params`
`424`	`424`	`zvol, err := s.apiClient.CreateZvol(ctx, createParams)`
`425`	`425`	`if err != nil {`
`426`	`426`	`timer.ObserveError()`
`427`		`- return nil, status.Errorf(codes.Internal, "Failed to create ZVOL %s: %v", params.zvolName, err)`
	`427`	`+ return nil, createVolumeError("Failed to create ZVOL "+params.zvolName, err)`
`428`	`428`	`}`
`429`	`429`
`430`	`430`	`klog.V(4).Infof("Created ZVOL: %s (ID: %s)", params.zvolName, zvol.ID)`
Original file line number	Diff line number	Diff line change
`@@ -389,7 +389,7 @@ func TestCreateISCSIVolume(t *testing.T) {`
`389`	`389`	`}`
`390`	`390`	`},`
`391`	`391`	`wantErr: true,`
`392`		`- wantCode: codes.Internal,`
	`392`	`+ wantCode: codes.ResourceExhausted,`
`393`	`393`	`},`
`394`	`394`	`}`
`395`	`395`
Original file line number	Diff line number	Diff line change
`@@ -423,7 +423,7 @@ func (s ControllerService) getOrCreateDataset(ctx context.Context, params nfsV`
`423`	`423`	`dataset, err := s.apiClient.CreateDataset(ctx, createParams)`
`424`	`424`	`if err != nil {`
`425`	`425`	`timer.ObserveError()`
`426`		`- return nil, status.Errorf(codes.Internal, "Failed to create dataset: %v", err)`
	`426`	`+ return nil, createVolumeError("Failed to create dataset", err)`
`427`	`427`	`}`
`428`	`428`
`429`	`429`	`klog.V(4).Infof("Created dataset: %s with mountpoint: %s", dataset.Name, dataset.Mountpoint)`
Original file line number	Diff line number	Diff line change
`@@ -620,7 +620,7 @@ func (s ControllerService) getOrCreateZVOL(ctx context.Context, params nvmeofV`
`620`	`620`	`zvol, err := s.apiClient.CreateZvol(ctx, createParams)`
`621`	`621`	`if err != nil {`
`622`	`622`	`timer.ObserveError()`
`623`		`- return nil, status.Errorf(codes.Internal, "Failed to create ZVOL: %v", err)`
	`623`	`+ return nil, createVolumeError("Failed to create ZVOL", err)`
`624`	`624`	`}`
`625`	`625`
`626`	`626`	`klog.V(4).Infof("Created ZVOL: %s (ID: %s)", zvol.Name, zvol.ID)`
Original file line number	Diff line number	Diff line change
`@@ -313,7 +313,7 @@ func TestCreateNVMeOFVolume(t *testing.T) {`
`313`	`313`	`}`
`314`	`314`	`},`
`315`	`315`	`wantErr: true,`
`316`		`- wantCode: codes.Internal,`
	`316`	`+ wantCode: codes.ResourceExhausted,`
`317`	`317`	`},`
`318`	`318`	`{`
`319`	`319`	`name: "subsystem creation failure with cleanup",`