feat: inline volume support

fix

fix

Author: andyzhangx

charts/latest/csi-driver-smb-v0.0.0.tgz

@@ -6,3 +6,6 @@ metadata:
 spec:
   attachRequired: false
   podInfoOnMount: true
+  volumeLifecycleModes:
+    - Persistent
+    - Ephemeral

charts/latest/csi-driver-smb/templates/csi-smb-driver.yaml

@@ -97,4 +97,29 @@ roleRef:
   kind: ClusterRole
   name: {{ .Values.rbac.name }}-external-resizer-role
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-{{ .Values.rbac.name }}-node-secret-role
+{{ include "smb.labels" . | indent 2 }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-{{ .Values.rbac.name }}-node-secret-binding
+{{ include "smb.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccount.node }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: csi-{{ .Values.rbac.name }}-node-secret-role
+  apiGroup: rbac.authorization.k8s.io
 {{ end }}

charts/latest/csi-driver-smb/templates/rbac-csi-smb.yaml

@@ -6,3 +6,6 @@ metadata:
 spec:
   attachRequired: false
   podInfoOnMount: true
+  volumeLifecycleModes:
+    - Persistent
+    - Ephemeral

deploy/csi-smb-driver.yaml

@@ -0,0 +1,27 @@
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: nginx-smb-inline-volume
+spec:
+  nodeSelector:
+    "kubernetes.io/os": linux
+  containers:
+    - image: mcr.microsoft.com/mirror/docker/library/nginx:1.23
+      name: nginx-smb
+      command:
+        - "/bin/bash"
+        - "-c"
+        - set -euo pipefail; while true; do echo $(date) >> /mnt/smb/outfile; sleep 1; done
+      volumeMounts:
+        - name: persistent-storage
+          mountPath: "/mnt/smb"
+          readOnly: false
+  volumes:
+    - name: persistent-storage
+      csi:
+        driver: smb.csi.k8s.io
+        volumeAttributes:
+          source: //smb-server.default.svc.cluster.local/share  # required
+          secretName: smbcreds  # required, secretNamespace is the same as the pod
+          mountOptions: "dir_mode=0777,file_mode=0777,cache=strict,actimeo=30,nosharesock"  # optional

deploy/example/nginx-pod-smb-inline-volume.yaml

@@ -87,3 +87,25 @@ roleRef:
   kind: ClusterRole
   name: smb-external-resizer-role
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-smb-node-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-smb-node-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-smb-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-smb-node-secret-role
+  apiGroup: rbac.authorization.k8s.io

deploy/rbac-csi-smb.yaml

@@ -36,13 +36,14 @@ import (
 
 	"golang.org/x/net/context"
 
-	volumehelper "github.com/kubernetes-csi/csi-driver-smb/pkg/util"
+	"github.com/kubernetes-csi/csi-driver-smb/pkg/util"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 )
 
 // NodePublishVolume mount the volume from staging to target path
-func (d *Driver) NodePublishVolume(_ context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
-	if req.GetVolumeCapability() == nil {
+func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
+	volCap := req.GetVolumeCapability()
+	if volCap == nil {
 		return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request")
 	}
 	volumeID := req.GetVolumeId()
@@ -55,6 +56,20 @@ func (d *Driver) NodePublishVolume(_ context.Context, req *csi.NodePublishVolume
 		return nil, status.Error(codes.InvalidArgument, "Target path not provided")
 	}
 
+	context := req.GetVolumeContext()
+	if context != nil && strings.EqualFold(context[ephemeralField], trueValue) {
+		// ephemeral volume
+		util.SetKeyValueInMap(context, secretNamespaceField, context[podNamespaceField])
+		klog.V(2).Infof("NodePublishVolume: ephemeral volume(%s) mount on %s", volumeID, target)
+		_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
+			StagingTargetPath: target,
+			VolumeContext:     context,
+			VolumeCapability:  volCap,
+			VolumeId:          volumeID,
+		})
+		return &csi.NodePublishVolumeResponse{}, err
+	}
+
 	source := req.GetStagingTargetPath()
 	if len(source) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "Staging target not provided")
@@ -110,7 +125,7 @@ func (d *Driver) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVo
 }
 
 // NodeStageVolume mount the volume to a staging path
-func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
 	volumeID := req.GetVolumeId()
 	if len(volumeID) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
@@ -132,7 +147,7 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ
 	secrets := req.GetSecrets()
 	gidPresent := checkGidPresentInMountFlags(mountFlags)
 
-	var source, subDir string
+	var source, subDir, secretName, secretNamespace string
 	subDirReplaceMap := map[string]string{}
 	for k, v := range context {
 		switch strings.ToLower(k) {
@@ -146,6 +161,10 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ
 			subDirReplaceMap[pvcNameMetadata] = v
 		case pvNameKey:
 			subDirReplaceMap[pvNameMetadata] = v
+		case secretNameField:
+			secretName = v
+		case secretNamespaceField:
+			secretNamespace = v
 		}
 	}
 
@@ -171,6 +190,15 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ
 		}
 	}
 
+	if (username == "" || password == "") && (secretName != "" && secretNamespace != "") {
+		klog.V(2).Infof("NodeStageVolume: getting username and password from secret %s in namespace %s", secretName, secretNamespace)
+		var err error
+		username, password, domain, err = d.GetUserNamePasswordFromSecret(ctx, secretName, secretNamespace)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("Error getting username and password from secret %s in namespace %s: %v", secretName, secretNamespace, err))
+		}
+	}
+
 	// in guest login, username and password options are not needed
 	requireUsernamePwdOption := !hasGuestMountOptions(mountFlags)
 
@@ -236,7 +264,7 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ
 			return Mount(d.mounter, source, targetPath, "cifs", mountOptions, sensitiveMountOptions, volumeID)
 		}
 		timeoutFunc := func() error { return fmt.Errorf("time out") }
-		if err := volumehelper.WaitUntilTimeout(90*time.Second, execFunc, timeoutFunc); err != nil {
+		if err := util.WaitUntilTimeout(90*time.Second, execFunc, timeoutFunc); err != nil {
 			return nil, status.Error(codes.Internal, fmt.Sprintf("volume(%s) mount %q on %q failed with %v", volumeID, source, targetPath, err))
 		}
 		klog.V(2).Infof("volume(%s) mount %q on %q succeeded", volumeID, source, targetPath)

pkg/smb/nodeserver.go

@@ -17,12 +17,22 @@ limitations under the License.
 package smb
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
 	mount "k8s.io/mount-utils"
 
@@ -40,8 +50,12 @@ const (
 	subDirField               = "subdir"
 	domainField               = "domain"
 	mountOptionsField         = "mountoptions"
+	secretNameField           = "secretname"
+	secretNamespaceField      = "secretnamespace"
 	paramOnDelete             = "ondelete"
 	defaultDomainName         = "AZURE"
+	ephemeralField            = "csi.storage.k8s.io/ephemeral"
+	podNamespaceField         = "csi.storage.k8s.io/pod.namespace"
 	pvcNameKey                = "csi.storage.k8s.io/pvc/name"
 	pvcNamespaceKey           = "csi.storage.k8s.io/pvc/namespace"
 	pvNameKey                 = "csi.storage.k8s.io/pv/name"
@@ -56,6 +70,7 @@ const (
 	dirMode                   = "dir_mode"
 	defaultFileMode           = "0777"
 	defaultDirMode            = "0777"
+	trueValue                 = "true"
 )
 
 var supportedOnDeleteValues = []string{"", "delete", retain, archive}
@@ -74,6 +89,7 @@ type DriverOptions struct {
 	DefaultOnDeletePolicy         string
 	RemoveArchivedVolumePath      bool
 	EnableWindowsHostProcess      bool
+	Kubeconfig                    string
 }
 
 // Driver implements all interfaces of CSI drivers
@@ -102,6 +118,8 @@ type Driver struct {
 	defaultOnDeletePolicy         string
 	removeArchivedVolumePath      bool
 	enableWindowsHostProcess      bool
+	kubeconfig                    string
+	kubeClient                    kubernetes.Interface
 }
 
 // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
@@ -116,6 +134,7 @@ func NewDriver(options *DriverOptions) *Driver {
 	driver.removeArchivedVolumePath = options.RemoveArchivedVolumePath
 	driver.workingMountDir = options.WorkingMountDir
 	driver.enableWindowsHostProcess = options.EnableWindowsHostProcess
+	driver.kubeconfig = options.Kubeconfig
 	driver.volumeLocks = newVolumeLocks()
 
 	driver.krb5CacheDirectory = options.Krb5CacheDirectory
@@ -138,6 +157,15 @@ func NewDriver(options *DriverOptions) *Driver {
 	if driver.volDeletionCache, err = azcache.NewTimedCache(time.Minute, getter, false); err != nil {
 		klog.Fatalf("%v", err)
 	}
+
+	kubeCfg, err := getKubeConfig(driver.kubeconfig, driver.enableWindowsHostProcess)
+	if err == nil && kubeCfg != nil {
+		if driver.kubeClient, err = kubernetes.NewForConfig(kubeCfg); err != nil {
+			klog.Warningf("NewForConfig failed with error: %v", err)
+		}
+	} else {
+		klog.Warningf("get kubeconfig(%s) failed with error: %v", driver.kubeconfig, err)
+	}
 	return &driver
 }
 
@@ -189,6 +217,24 @@ func (d *Driver) Run(endpoint, _ string, testMode bool) {
 	s.Wait()
 }
 
+// GetUserNamePasswordFromSecret get storage account key from k8s secret
+// return <username, password, domain, error>
+func (d *Driver) GetUserNamePasswordFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, error) {
+	if d.kubeClient == nil {
+		return "", "", "", fmt.Errorf("could not username and password from secret(%s): KubeClient is nil", secretName)
+	}
+
+	secret, err := d.kubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
+	if err != nil {
+		return "", "", "", fmt.Errorf("could not get secret(%v): %v", secretName, err)
+	}
+
+	username := strings.TrimSpace(string(secret.Data[usernameField][:]))
+	password := strings.TrimSpace(string(secret.Data[passwordField][:]))
+	domain := strings.TrimSpace(string(secret.Data[domainField][:]))
+	return username, password, domain, nil
+}
+
 func IsCorruptedDir(dir string) bool {
 	_, pathErr := mount.PathExists(dir)
 	return pathErr != nil && mount.IsCorruptedMnt(pathErr)
@@ -279,3 +325,61 @@ func getRootDir(path string) string {
 	parts := strings.Split(path, "/")
 	return parts[0]
 }
+
+func getKubeConfig(kubeconfig string, enableWindowsHostProcess bool) (config *rest.Config, err error) {
+	if kubeconfig != "" {
+		if config, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
+			return nil, err
+		}
+	} else {
+		if config, err = inClusterConfig(enableWindowsHostProcess); err != nil {
+			return nil, err
+		}
+	}
+	return config, err
+}
+
+// inClusterConfig is copied from https://github.com/kubernetes/client-go/blob/b46677097d03b964eab2d67ffbb022403996f4d4/rest/config.go#L507-L541
+// When using Windows HostProcess containers, the path "/var/run/secrets/kubernetes.io/serviceaccount/" is under host, not container.
+// Then the token and ca.crt files would be not found.
+// An environment variable $CONTAINER_SANDBOX_MOUNT_POINT is set upon container creation and provides the absolute host path to the container volume.
+// See https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts for more details.
+func inClusterConfig(enableWindowsHostProcess bool) (*rest.Config, error) {
+	var (
+		tokenFile  = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+		rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+	)
+	if enableWindowsHostProcess {
+		containerSandboxMountPath := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT")
+		if len(containerSandboxMountPath) == 0 {
+			return nil, errors.New("unable to load in-cluster configuration, containerSandboxMountPath must be defined")
+		}
+		tokenFile = filepath.Join(containerSandboxMountPath, tokenFile)
+		rootCAFile = filepath.Join(containerSandboxMountPath, rootCAFile)
+	}
+
+	host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
+	if len(host) == 0 || len(port) == 0 {
+		return nil, rest.ErrNotInCluster
+	}
+
+	token, err := os.ReadFile(tokenFile)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsClientConfig := rest.TLSClientConfig{}
+
+	if _, err := certutil.NewPool(rootCAFile); err != nil {
+		klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err)
+	} else {
+		tlsClientConfig.CAFile = rootCAFile
+	}
+
+	return &rest.Config{
+		Host:            "https://" + net.JoinHostPort(host, port),
+		TLSClientConfig: tlsClientConfig,
+		BearerToken:     string(token),
+		BearerTokenFile: tokenFile,
+	}, nil
+}