diff --git a/charts/latest/csi-driver-smb-v0.0.0.tgz b/charts/latest/csi-driver-smb-v0.0.0.tgz index dbd4950fb80..d04f15ebea9 100644 Binary files a/charts/latest/csi-driver-smb-v0.0.0.tgz and b/charts/latest/csi-driver-smb-v0.0.0.tgz differ diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-driver.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-driver.yaml index 16094379923..6655aa32500 100755 --- a/charts/latest/csi-driver-smb/templates/csi-smb-driver.yaml +++ b/charts/latest/csi-driver-smb/templates/csi-smb-driver.yaml @@ -6,3 +6,8 @@ metadata: spec: attachRequired: false podInfoOnMount: true + volumeLifecycleModes: + - Persistent + {{- if .Values.feature.enableInlineVolume }} + - Ephemeral + {{- end }} diff --git a/charts/latest/csi-driver-smb/templates/rbac-csi-smb.yaml b/charts/latest/csi-driver-smb/templates/rbac-csi-smb.yaml index 3e13eed752e..6d89878eff2 100755 --- a/charts/latest/csi-driver-smb/templates/rbac-csi-smb.yaml +++ b/charts/latest/csi-driver-smb/templates/rbac-csi-smb.yaml @@ -97,4 +97,30 @@ roleRef: kind: ClusterRole name: {{ .Values.rbac.name }}-external-resizer-role apiGroup: rbac.authorization.k8s.io +--- +{{- if .Values.feature.enableInlineVolume }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-role +{{ include "smb.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-binding +{{ include "smb.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-node-secret-role + apiGroup: rbac.authorization.k8s.io +{{- end }} {{ end }} diff --git a/charts/latest/csi-driver-smb/values.yaml b/charts/latest/csi-driver-smb/values.yaml index 4a7abb3e975..7cda0b0f43b 100755 --- a/charts/latest/csi-driver-smb/values.yaml +++ b/charts/latest/csi-driver-smb/values.yaml @@ -39,6 +39,7 @@ driver: feature: enableGetVolumeStats: true + enableInlineVolume: true controller: name: csi-smb-controller diff --git a/deploy/csi-smb-driver.yaml b/deploy/csi-smb-driver.yaml index 6450d22b3e9..94573ff15e1 100644 --- a/deploy/csi-smb-driver.yaml +++ b/deploy/csi-smb-driver.yaml @@ -6,3 +6,6 @@ metadata: spec: attachRequired: false podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral diff --git a/deploy/example/nginx-pod-smb-inline-volume.yaml b/deploy/example/nginx-pod-smb-inline-volume.yaml new file mode 100644 index 00000000000..f1b51db1720 --- /dev/null +++ b/deploy/example/nginx-pod-smb-inline-volume.yaml @@ -0,0 +1,27 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: nginx-smb-inline-volume +spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - image: mcr.microsoft.com/mirror/docker/library/nginx:1.23 + name: nginx-smb + command: + - "/bin/bash" + - "-c" + - set -euo pipefail; while true; do echo $(date) >> /mnt/smb/outfile; sleep 1; done + volumeMounts: + - name: persistent-storage + mountPath: "/mnt/smb" + readOnly: false + volumes: + - name: persistent-storage + csi: + driver: smb.csi.k8s.io + volumeAttributes: + source: //smb-server.default.svc.cluster.local/share # required + secretName: smbcreds # required, secretNamespace is the same as the pod + mountOptions: "dir_mode=0777,file_mode=0777,cache=strict,actimeo=30,nosharesock" # optional diff --git a/deploy/rbac-csi-smb.yaml b/deploy/rbac-csi-smb.yaml index 248a61c7b28..490f3dcf9eb 100644 --- a/deploy/rbac-csi-smb.yaml +++ b/deploy/rbac-csi-smb.yaml @@ -87,3 +87,25 @@ roleRef: kind: ClusterRole name: smb-external-resizer-role apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-smb-node-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-smb-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-smb-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-smb-node-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/smb/nodeserver.go b/pkg/smb/nodeserver.go index a1ca4de8642..0528003ea01 100644 --- a/pkg/smb/nodeserver.go +++ b/pkg/smb/nodeserver.go @@ -36,13 +36,14 @@ import ( "golang.org/x/net/context" - volumehelper "github.com/kubernetes-csi/csi-driver-smb/pkg/util" + "github.com/kubernetes-csi/csi-driver-smb/pkg/util" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" ) // NodePublishVolume mount the volume from staging to target path -func (d *Driver) NodePublishVolume(_ context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if req.GetVolumeCapability() == nil { +func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + volCap := req.GetVolumeCapability() + if volCap == nil { return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") } volumeID := req.GetVolumeId() @@ -55,6 +56,20 @@ func (d *Driver) NodePublishVolume(_ context.Context, req *csi.NodePublishVolume return nil, status.Error(codes.InvalidArgument, "Target path not provided") } + context := req.GetVolumeContext() + if context != nil && strings.EqualFold(context[ephemeralField], trueValue) { + // ephemeral volume + util.SetKeyValueInMap(context, secretNamespaceField, context[podNamespaceField]) + klog.V(2).Infof("NodePublishVolume: ephemeral volume(%s) mount on %s", volumeID, target) + _, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ + StagingTargetPath: target, + VolumeContext: context, + VolumeCapability: volCap, + VolumeId: volumeID, + }) + return &csi.NodePublishVolumeResponse{}, err + } + source := req.GetStagingTargetPath() if len(source) == 0 { return nil, status.Error(codes.InvalidArgument, "Staging target not provided") @@ -110,7 +125,7 @@ func (d *Driver) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVo } // NodeStageVolume mount the volume to a staging path -func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { +func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { volumeID := req.GetVolumeId() if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") @@ -132,7 +147,8 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ secrets := req.GetSecrets() gidPresent := checkGidPresentInMountFlags(mountFlags) - var source, subDir string + var source, subDir, secretName, secretNamespace, ephemeralVolMountOptions string + var ephemeralVol bool subDirReplaceMap := map[string]string{} for k, v := range context { switch strings.ToLower(k) { @@ -146,6 +162,14 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ subDirReplaceMap[pvcNameMetadata] = v case pvNameKey: subDirReplaceMap[pvNameMetadata] = v + case secretNameField: + secretName = v + case secretNamespaceField: + secretNamespace = v + case ephemeralField: + ephemeralVol = strings.EqualFold(v, trueValue) + case mountOptionsField: + ephemeralVolMountOptions = v } } @@ -171,8 +195,20 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ } } + if ephemeralVol { + mountFlags = strings.Split(ephemeralVolMountOptions, ",") + } + // in guest login, username and password options are not needed requireUsernamePwdOption := !hasGuestMountOptions(mountFlags) + if ephemeralVol && requireUsernamePwdOption { + klog.V(2).Infof("NodeStageVolume: getting username and password from secret %s in namespace %s", secretName, secretNamespace) + var err error + username, password, domain, err = d.GetUserNamePasswordFromSecret(ctx, secretName, secretNamespace) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("Error getting username and password from secret %s in namespace %s: %v", secretName, secretNamespace, err)) + } + } var mountOptions, sensitiveMountOptions []string if runtime.GOOS == "windows" { @@ -236,7 +272,7 @@ func (d *Driver) NodeStageVolume(_ context.Context, req *csi.NodeStageVolumeRequ return Mount(d.mounter, source, targetPath, "cifs", mountOptions, sensitiveMountOptions, volumeID) } timeoutFunc := func() error { return fmt.Errorf("time out") } - if err := volumehelper.WaitUntilTimeout(90*time.Second, execFunc, timeoutFunc); err != nil { + if err := util.WaitUntilTimeout(90*time.Second, execFunc, timeoutFunc); err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("volume(%s) mount %q on %q failed with %v", volumeID, source, targetPath, err)) } klog.V(2).Infof("volume(%s) mount %q on %q succeeded", volumeID, source, targetPath) diff --git a/pkg/smb/nodeserver_test.go b/pkg/smb/nodeserver_test.go index fa52417194c..4b87fe1a987 100644 --- a/pkg/smb/nodeserver_test.go +++ b/pkg/smb/nodeserver_test.go @@ -413,6 +413,37 @@ func TestNodePublishVolume(t *testing.T) { Readonly: true}, expectedErr: testutil.TestError{}, }, + { + desc: "[Error] failed to create ephemeral Volume", + req: &csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap}, + VolumeId: "vol_1", + TargetPath: targetTest, + StagingTargetPath: sourceTest, + Readonly: true, + VolumeContext: map[string]string{ephemeralField: "true"}, + }, + expectedErr: testutil.TestError{ + DefaultError: status.Error(codes.InvalidArgument, "source field is missing, current context: map[csi.storage.k8s.io/ephemeral:true secretnamespace:]"), + }, + }, + { + desc: "[error] failed request with ephemeral Volume", + req: &csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap}, + VolumeId: "vol_1", + TargetPath: targetTest, + StagingTargetPath: sourceTest, + Readonly: true, + VolumeContext: map[string]string{ + ephemeralField: "true", + sourceField: "source", + podNamespaceField: "podnamespace", + }, + }, + skipOnWindows: true, + expectedErr: testutil.TestError{ + DefaultError: status.Error(codes.Internal, "Error getting username and password from secret in namespace podnamespace: could not username and password from secret(): KubeClient is nil"), + }, + }, } // Setup diff --git a/pkg/smb/smb.go b/pkg/smb/smb.go index a4e27abf061..d0f2365708b 100644 --- a/pkg/smb/smb.go +++ b/pkg/smb/smb.go @@ -17,12 +17,22 @@ limitations under the License. package smb import ( + "context" + "errors" "fmt" + "net" + "os" + "path/filepath" "strings" "time" "github.com/container-storage-interface/spec/lib/go/csi" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + certutil "k8s.io/client-go/util/cert" "k8s.io/klog/v2" mount "k8s.io/mount-utils" @@ -40,8 +50,12 @@ const ( subDirField = "subdir" domainField = "domain" mountOptionsField = "mountoptions" + secretNameField = "secretname" + secretNamespaceField = "secretnamespace" paramOnDelete = "ondelete" defaultDomainName = "AZURE" + ephemeralField = "csi.storage.k8s.io/ephemeral" + podNamespaceField = "csi.storage.k8s.io/pod.namespace" pvcNameKey = "csi.storage.k8s.io/pvc/name" pvcNamespaceKey = "csi.storage.k8s.io/pvc/namespace" pvNameKey = "csi.storage.k8s.io/pv/name" @@ -56,6 +70,7 @@ const ( dirMode = "dir_mode" defaultFileMode = "0777" defaultDirMode = "0777" + trueValue = "true" ) var supportedOnDeleteValues = []string{"", "delete", retain, archive} @@ -74,6 +89,7 @@ type DriverOptions struct { DefaultOnDeletePolicy string RemoveArchivedVolumePath bool EnableWindowsHostProcess bool + Kubeconfig string } // Driver implements all interfaces of CSI drivers @@ -102,6 +118,8 @@ type Driver struct { defaultOnDeletePolicy string removeArchivedVolumePath bool enableWindowsHostProcess bool + kubeconfig string + kubeClient kubernetes.Interface } // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & @@ -116,6 +134,7 @@ func NewDriver(options *DriverOptions) *Driver { driver.removeArchivedVolumePath = options.RemoveArchivedVolumePath driver.workingMountDir = options.WorkingMountDir driver.enableWindowsHostProcess = options.EnableWindowsHostProcess + driver.kubeconfig = options.Kubeconfig driver.volumeLocks = newVolumeLocks() driver.krb5CacheDirectory = options.Krb5CacheDirectory @@ -138,6 +157,15 @@ func NewDriver(options *DriverOptions) *Driver { if driver.volDeletionCache, err = azcache.NewTimedCache(time.Minute, getter, false); err != nil { klog.Fatalf("%v", err) } + + kubeCfg, err := getKubeConfig(driver.kubeconfig, driver.enableWindowsHostProcess) + if err == nil && kubeCfg != nil { + if driver.kubeClient, err = kubernetes.NewForConfig(kubeCfg); err != nil { + klog.Warningf("NewForConfig failed with error: %v", err) + } + } else { + klog.Warningf("get kubeconfig(%s) failed with error: %v", driver.kubeconfig, err) + } return &driver } @@ -189,6 +217,24 @@ func (d *Driver) Run(endpoint, _ string, testMode bool) { s.Wait() } +// GetUserNamePasswordFromSecret get storage account key from k8s secret +// return +func (d *Driver) GetUserNamePasswordFromSecret(ctx context.Context, secretName, secretNamespace string) (string, string, string, error) { + if d.kubeClient == nil { + return "", "", "", fmt.Errorf("could not username and password from secret(%s): KubeClient is nil", secretName) + } + + secret, err := d.kubeClient.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return "", "", "", fmt.Errorf("could not get secret(%v): %v", secretName, err) + } + + username := strings.TrimSpace(string(secret.Data[usernameField][:])) + password := strings.TrimSpace(string(secret.Data[passwordField][:])) + domain := strings.TrimSpace(string(secret.Data[domainField][:])) + return username, password, domain, nil +} + func IsCorruptedDir(dir string) bool { _, pathErr := mount.PathExists(dir) return pathErr != nil && mount.IsCorruptedMnt(pathErr) @@ -279,3 +325,61 @@ func getRootDir(path string) string { parts := strings.Split(path, "/") return parts[0] } + +func getKubeConfig(kubeconfig string, enableWindowsHostProcess bool) (config *rest.Config, err error) { + if kubeconfig != "" { + if config, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil { + return nil, err + } + } else { + if config, err = inClusterConfig(enableWindowsHostProcess); err != nil { + return nil, err + } + } + return config, err +} + +// inClusterConfig is copied from https://github.com/kubernetes/client-go/blob/b46677097d03b964eab2d67ffbb022403996f4d4/rest/config.go#L507-L541 +// When using Windows HostProcess containers, the path "/var/run/secrets/kubernetes.io/serviceaccount/" is under host, not container. +// Then the token and ca.crt files would be not found. +// An environment variable $CONTAINER_SANDBOX_MOUNT_POINT is set upon container creation and provides the absolute host path to the container volume. +// See https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts for more details. +func inClusterConfig(enableWindowsHostProcess bool) (*rest.Config, error) { + var ( + tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" + rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ) + if enableWindowsHostProcess { + containerSandboxMountPath := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + if len(containerSandboxMountPath) == 0 { + return nil, errors.New("unable to load in-cluster configuration, containerSandboxMountPath must be defined") + } + tokenFile = filepath.Join(containerSandboxMountPath, tokenFile) + rootCAFile = filepath.Join(containerSandboxMountPath, rootCAFile) + } + + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{} + + if _, err := certutil.NewPool(rootCAFile); err != nil { + klog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + } else { + tlsClientConfig.CAFile = rootCAFile + } + + return &rest.Config{ + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} diff --git a/pkg/smb/smb_test.go b/pkg/smb/smb_test.go index 090652a7966..21761cbbd49 100644 --- a/pkg/smb/smb_test.go +++ b/pkg/smb/smb_test.go @@ -420,3 +420,112 @@ func TestGetRootPath(t *testing.T) { } } } + +func TestGetKubeConfig(t *testing.T) { + // skip for now as this is very flaky on Windows + //skipIfTestingOnWindows(t) + emptyKubeConfig := "empty-Kube-Config" + validKubeConfig := "valid-Kube-Config" + fakeContent := ` +apiVersion: v1 +clusters: +- cluster: + server: https://localhost:8080 + name: foo-cluster +contexts: +- context: + cluster: foo-cluster + user: foo-user + namespace: bar + name: foo-context +current-context: foo-context +kind: Config +users: +- name: foo-user + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - arg-1 + - arg-2 + command: foo-command +` + err := createTestFile(emptyKubeConfig) + if err != nil { + t.Error(err) + } + defer func() { + if err := os.Remove(emptyKubeConfig); err != nil { + t.Error(err) + } + }() + + err = createTestFile(validKubeConfig) + if err != nil { + t.Error(err) + } + defer func() { + if err := os.Remove(validKubeConfig); err != nil { + t.Error(err) + } + }() + + if err := os.WriteFile(validKubeConfig, []byte(fakeContent), 0666); err != nil { + t.Error(err) + } + + os.Setenv("CONTAINER_SANDBOX_MOUNT_POINT", "C:\\var\\lib\\kubelet\\pods\\12345678-1234-1234-1234-123456789012") + defer os.Unsetenv("CONTAINER_SANDBOX_MOUNT_POINT") + + tests := []struct { + desc string + kubeconfig string + enableWindowsHostProcess bool + expectError bool + envVariableHasConfig bool + envVariableConfigIsValid bool + }{ + { + desc: "[success] valid kube config passed", + kubeconfig: validKubeConfig, + enableWindowsHostProcess: false, + expectError: false, + envVariableHasConfig: false, + envVariableConfigIsValid: false, + }, + { + desc: "[failure] invalid kube config passed", + kubeconfig: emptyKubeConfig, + enableWindowsHostProcess: false, + expectError: true, + envVariableHasConfig: false, + envVariableConfigIsValid: false, + }, + { + desc: "[failure] empty Kubeconfig under container sandbox mount path", + kubeconfig: "", + enableWindowsHostProcess: true, + expectError: true, + envVariableHasConfig: false, + envVariableConfigIsValid: false, + }, + } + + for _, test := range tests { + _, err := getKubeConfig(test.kubeconfig, test.enableWindowsHostProcess) + receiveError := (err != nil) + if test.expectError != receiveError { + t.Errorf("desc: %s,\n input: %q, GetCloudProvider err: %v, expectErr: %v", test.desc, test.kubeconfig, err, test.expectError) + } + } +} + +func createTestFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return nil +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 638e8391bf3..5f50d38a91c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -17,10 +17,12 @@ limitations under the License. package util import ( - "k8s.io/klog/v2" "os" "os/exec" + "strings" "time" + + "k8s.io/klog/v2" ) const MaxPathLengthWindows = 260 @@ -65,3 +67,18 @@ func RunPowershellCmd(command string, envs ...string) ([]byte, error) { klog.V(6).Infof("Executing command: %q", cmd.String()) return cmd.CombinedOutput() } + +// SetKeyValueInMap set key/value pair in map +// key in the map is case insensitive, if key already exists, overwrite existing value +func SetKeyValueInMap(m map[string]string, key, value string) { + if m == nil { + return + } + for k := range m { + if strings.EqualFold(k, key) { + m[k] = value + return + } + } + m[key] = value +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 718b59a563f..1663e96df42 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -18,6 +18,7 @@ package util import ( "fmt" + "reflect" "testing" "time" ) @@ -73,3 +74,57 @@ func TestWaitUntilTimeout(t *testing.T) { } } } + +func TestSetKeyValueInMap(t *testing.T) { + tests := []struct { + desc string + m map[string]string + key string + value string + expected map[string]string + }{ + { + desc: "nil map", + key: "key", + value: "value", + }, + { + desc: "empty map", + m: map[string]string{}, + key: "key", + value: "value", + expected: map[string]string{"key": "value"}, + }, + { + desc: "non-empty map", + m: map[string]string{"k": "v"}, + key: "key", + value: "value", + expected: map[string]string{ + "k": "v", + "key": "value", + }, + }, + { + desc: "same key already exists", + m: map[string]string{"subDir": "value2"}, + key: "subDir", + value: "value", + expected: map[string]string{"subDir": "value"}, + }, + { + desc: "case insensitive key already exists", + m: map[string]string{"subDir": "value2"}, + key: "subdir", + value: "value", + expected: map[string]string{"subDir": "value"}, + }, + } + + for _, test := range tests { + SetKeyValueInMap(test.m, test.key, test.value) + if !reflect.DeepEqual(test.m, test.expected) { + t.Errorf("test[%s]: unexpected output: %v, expected result: %v", test.desc, test.m, test.expected) + } + } +} diff --git a/test/e2e/dynamic_provisioning_test.go b/test/e2e/dynamic_provisioning_test.go index b269cd55cef..3be0cc99b38 100644 --- a/test/e2e/dynamic_provisioning_test.go +++ b/test/e2e/dynamic_provisioning_test.go @@ -533,4 +533,49 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { } test.Run(ctx, cs, ns) }) + + ginkgo.It("should create an CSI inline volume", func(ctx ginkgo.SpecContext) { + if winServerVer == "windows-2022" && !isWindowsHostProcessDeployment { + ginkgo.Skip("Skip inline volume test on Windows Server 2022") + } + + secretName := "smbcreds" + ginkgo.By(fmt.Sprintf("creating secret %s in namespace %s", secretName, ns.Name)) + tsecret := testsuites.CopyTestSecret(ctx, cs, "default", ns, defaultSmbSecretName) + tsecret.Create(ctx) + defer tsecret.Cleanup(ctx) + + pods := []testsuites.PodDetails{ + { + Cmd: convertToPowershellCommandIfNecessary("echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data"), + Volumes: []testsuites.VolumeDetails{ + { + ClaimSize: "100Gi", + MountOptions: []string{ + "uid=0", + "gid=0", + "mfsymlinks", + "cache=strict", + "nosharesock", + }, + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + }, + }, + IsWindows: isWindowsCluster, + WinServerVer: winServerVer, + }, + } + + test := testsuites.DynamicallyProvisionedInlineVolumeTest{ + CSIDriver: testDriver, + Pods: pods, + Source: defaultStorageClassParameters["source"], + SecretName: defaultSmbSecretName, + ReadOnly: false, + } + test.Run(ctx, cs, ns) + }) }) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 17c931cd921..ca299fa8f76 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -50,6 +50,7 @@ const ( defaultSmbSource = "//smb-server.default.svc.cluster.local/share" defaultSmbSecretName = "smbcreds" defaultSmbSecretNamespace = "default" + accountNameForTest = "YW5keXNzZGZpbGUK" ) var ( @@ -173,7 +174,7 @@ var _ = ginkgo.BeforeSuite(func() { } if isWindowsHostProcessDeployment { - decodedBytes, err := base64.StdEncoding.DecodeString("YW5keXNzZGZpbGUK") + decodedBytes, err := base64.StdEncoding.DecodeString(accountNameForTest) if err != nil { log.Printf("Error decoding base64 string: %v\n", err) return diff --git a/test/e2e/testsuites/dynamically_provisioned_inline_volume.go b/test/e2e/testsuites/dynamically_provisioned_inline_volume.go new file mode 100644 index 00000000000..33be4ea614d --- /dev/null +++ b/test/e2e/testsuites/dynamically_provisioned_inline_volume.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + + "github.com/kubernetes-csi/csi-driver-smb/test/e2e/driver" + + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" +) + +// DynamicallyProvisionedInlineVolumeTest will provision required Source, SecretName +// Waiting for the PV provisioner to create an inline volume +// Testing if the Pod(s) Cmd is run with a 0 exit code +type DynamicallyProvisionedInlineVolumeTest struct { + CSIDriver driver.DynamicPVTestDriver + Pods []PodDetails + Source string + SecretName string + ReadOnly bool +} + +func (t *DynamicallyProvisionedInlineVolumeTest) Run(ctx context.Context, client clientset.Interface, namespace *v1.Namespace) { + for _, pod := range t.Pods { + tpod, cleanup := pod.SetupWithCSIInlineVolumes(client, namespace, t.Source, t.SecretName, t.ReadOnly) + + // defer must be called here for resources not get removed before using them + for i := range cleanup { + defer cleanup[i]() + } + + ginkgo.By("deploying the pod") + tpod.Create(ctx) + defer tpod.Cleanup(ctx) + ginkgo.By("checking that the pods command exits with no error") + tpod.WaitForSuccess(ctx) + } +} diff --git a/test/e2e/testsuites/specs.go b/test/e2e/testsuites/specs.go index 30e744e898a..224ca50aef0 100644 --- a/test/e2e/testsuites/specs.go +++ b/test/e2e/testsuites/specs.go @@ -135,6 +135,15 @@ func (pod *PodDetails) SetupWithPreProvisionedVolumes(ctx context.Context, clien return tpod, cleanupFuncs } +func (pod *PodDetails) SetupWithCSIInlineVolumes(client clientset.Interface, namespace *v1.Namespace, source, secretName string, readOnly bool) (*TestPod, []func()) { + tpod := NewTestPod(client, namespace, pod.Cmd, pod.IsWindows, pod.WinServerVer) + cleanupFuncs := make([]func(), 0) + for n, v := range pod.Volumes { + tpod.SetupCSIInlineVolume(fmt.Sprintf("%s%d", v.VolumeMount.NameGenerate, n+1), fmt.Sprintf("%s%d", v.VolumeMount.MountPathGenerate, n+1), source, secretName, readOnly) + } + return tpod, cleanupFuncs +} + func (pod *PodDetails) SetupDeployment(ctx context.Context, client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver, storageClassParameters map[string]string) (*TestDeployment, []func(ctx context.Context)) { cleanupFuncs := make([]func(ctx context.Context), 0) volume := pod.Volumes[0] diff --git a/test/e2e/testsuites/testsuites.go b/test/e2e/testsuites/testsuites.go index 92e17ff8eca..dda6afa81e4 100644 --- a/test/e2e/testsuites/testsuites.go +++ b/test/e2e/testsuites/testsuites.go @@ -49,6 +49,7 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" imageutils "k8s.io/kubernetes/test/utils/image" + "k8s.io/utils/ptr" ) const ( @@ -637,6 +638,31 @@ func (t *TestPod) SetupRawBlockVolume(pvc *v1.PersistentVolumeClaim, name, devic t.pod.Spec.Volumes = append(t.pod.Spec.Volumes, volume) } +func (t *TestPod) SetupCSIInlineVolume(name, mountPath, source, secretName string, readOnly bool) { + volumeMount := v1.VolumeMount{ + Name: name, + MountPath: mountPath, + ReadOnly: readOnly, + } + t.pod.Spec.Containers[0].VolumeMounts = append(t.pod.Spec.Containers[0].VolumeMounts, volumeMount) + + volume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + CSI: &v1.CSIVolumeSource{ + Driver: smb.DefaultDriverName, + VolumeAttributes: map[string]string{ + "source": source, + "secretName": secretName, + "mountOptions": "dir_mode=0777,file_mode=0777,cache=strict,actimeo=30,nosharesock", + }, + ReadOnly: ptr.To(readOnly), + }, + }, + } + t.pod.Spec.Volumes = append(t.pod.Spec.Volumes, volume) +} + func (t *TestPod) SetNodeSelector(nodeSelector map[string]string) { t.pod.Spec.NodeSelector = nodeSelector } @@ -691,6 +717,25 @@ func NewTestSecret(c clientset.Interface, ns *v1.Namespace, name string, data ma } } +func CopyTestSecret(ctx context.Context, c clientset.Interface, sourceNamespace string, targetNamespace *v1.Namespace, secretName string) *TestSecret { + secret, err := c.CoreV1().Secrets(sourceNamespace).Get(ctx, secretName, metav1.GetOptions{}) + framework.ExpectNoError(err) + + return &TestSecret{ + client: c, + namespace: targetNamespace, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: targetNamespace.Name, + }, + StringData: secret.StringData, + Data: secret.Data, + Type: v1.SecretTypeOpaque, + }, + } +} + func (t *TestSecret) Create(ctx context.Context) { var err error t.secret, err = t.client.CoreV1().Secrets(t.namespace.Name).Create(ctx, t.secret, metav1.CreateOptions{})