add e2e test for sas token and key vault suppport #103

Author: cvvz

go.mod

@@ -1,9 +1,10 @@
 module sigs.k8s.io/blob-csi-driver
 
-go 1.17
+go 1.18
 
 require (
 	github.com/Azure/azure-sdk-for-go v65.0.0+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0
 	github.com/Azure/go-autorest/autorest v0.11.27
 	github.com/Azure/go-autorest/autorest/adal v0.9.20
 	github.com/Azure/go-autorest/autorest/to v0.4.0
@@ -18,7 +19,7 @@ require (
 	github.com/pborman/uuid v1.2.0
 	github.com/pelletier/go-toml v1.9.4
 	github.com/stretchr/testify v1.7.1
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
 	google.golang.org/grpc v1.42.0
 	google.golang.org/protobuf v1.27.1
 	k8s.io/api v0.24.1
@@ -34,12 +35,19 @@ require (
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.0.0
+)
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect
 	github.com/aws/aws-sdk-go v1.38.49 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
@@ -53,6 +61,7 @@ require (
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
 	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
@@ -63,6 +72,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -71,6 +81,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/opencontainers/selinux v1.8.2 // indirect
+	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
@@ -89,7 +100,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/trace v0.20.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
+	golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect

go.sum

@@ -16,10 +16,12 @@ export set AZURE_CLIENT_SECRET=
 # If the the test is not for the public Azure, e.g. Azure China Cloud, then you need to set AZURE_CLOUD_NAME and AZURE_LOCATION.
 # For Azure Stack Clound, you need to set AZURE_ENVIRONMENT_FILEPATH for your cloud environment.
 # If you have an existing resource group created for the test, then you need to set variable AZURE_RESOURCE_GROUP.
+# If you are going to test reading storage account key(or sastoken) from Azure Key Vault, then you need to set variable AZURE_OBJECT_ID as the objectId of agentpool Managed Identity 
 export set AZURE_CLOUD_NAME=
 export set AZURE_LOCATION=
 export set AZURE_ENVIRONMENT_FILEPATH=
 export set AZURE_RESOURCE_GROUP=
+export set AZURE_OBJECT_ID=
 ```
 
 ### Run test

test/e2e/README.md

@@ -250,6 +250,46 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Pre-Provisioned", func() {
 		}
 		test.Run(cs, ns)
 	})
+
+	ginkgo.It("should use Key Vault", func() {
+		req := makeCreateVolumeReq("pre-provisioned-key-vault", ns.Name)
+		resp, err := blobDriver.CreateVolume(context.Background(), req)
+		if err != nil {
+			ginkgo.Fail(fmt.Sprintf("create volume error: %v", err))
+		}
+		volumeID = resp.Volume.VolumeId
+		ginkgo.By(fmt.Sprintf("Successfully provisioned blob volume: %q\n", volumeID))
+
+		volumeSize := fmt.Sprintf("%dGi", defaultVolumeSize)
+		reclaimPolicy := v1.PersistentVolumeReclaimRetain
+		volumeBindingMode := storagev1.VolumeBindingImmediate
+
+		pods := []testsuites.PodDetails{
+			{
+				Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
+				Volumes: []testsuites.VolumeDetails{
+					{
+						VolumeID:          volumeID,
+						FSType:            "ext4",
+						ClaimSize:         volumeSize,
+						ReclaimPolicy:     &reclaimPolicy,
+						VolumeBindingMode: &volumeBindingMode,
+						VolumeMount: testsuites.VolumeMountDetails{
+							NameGenerate:      "test-volume-",
+							MountPathGenerate: "/mnt/test-",
+						},
+					},
+				},
+			},
+		}
+
+		test := testsuites.PreProvisionedKeyVaultTest{
+			CSIDriver: testDriver,
+			Pods:      pods,
+			Driver:    blobDriver,
+		}
+		test.Run(cs, ns)
+	})
 })
 
 func makeCreateVolumeReq(volumeName, secretNamespace string) *csi.CreateVolumeRequest {

test/e2e/pre_provisioning_test.go

@@ -0,0 +1,218 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault"
+	"github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"sigs.k8s.io/blob-csi-driver/pkg/blob"
+	"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
+	"sigs.k8s.io/blob-csi-driver/test/utils/credentials"
+)
+
+var (
+	subscriptionID    string
+	resourceGroupName string
+	location          string
+	vaultName         string
+	TenantID          string
+	ObjectID          string
+)
+
+// PreProvisionedKeyVaultTest will provision required PV(s), PVC(s) and Pod(s)
+// Testing that the Pod(s) can be created successfully with provided Key Vault
+// which is used to store storage account name and key(or sastoken)
+type PreProvisionedKeyVaultTest struct {
+	CSIDriver driver.PreProvisionedVolumeTestDriver
+	Pods      []PodDetails
+	Driver    *blob.Driver
+}
+
+func (t *PreProvisionedKeyVaultTest) Run(client clientset.Interface, namespace *v1.Namespace) {
+	e2eCred, err := credentials.ParseAzureCredentialFile()
+	framework.ExpectNoError(err, fmt.Sprintf("Error ParseAzureCredentialFile: %v", err))
+
+	subscriptionID = e2eCred.SubscriptionID
+	resourceGroupName = e2eCred.ResourceGroup
+	location = e2eCred.Location
+	TenantID = e2eCred.TenantID
+	ObjectID = os.Getenv("AZURE_OBJECT_ID")
+	framework.ExpectNotEqual(len(ObjectID), 0, "env AZURE_OBJECT_ID must be set")
+	vaultName = "blobcsidriver-kv-test"
+
+	for _, pod := range t.Pods {
+		for n, volume := range pod.Volumes {
+			accountName, accountKey, _, containerName, err := t.Driver.GetStorageAccountAndContainer(context.TODO(), volume.VolumeID, nil, nil)
+			framework.ExpectNoError(err, fmt.Sprintf("Error GetStorageAccountAndContainer from volumeID(%s): %v", volume.VolumeID, err))
+
+			azureCred, err := azidentity.NewDefaultAzureCredential(nil)
+			framework.ExpectNoError(err)
+
+			vault, err := createVault(context.TODO(), azureCred)
+			framework.ExpectNoError(err)
+			defer cleanVault(context.TODO(), azureCred)
+
+			accountKeySecret, err := createSecret(context.TODO(), azureCred, accountName+"-key", accountKey)
+			framework.ExpectNoError(err)
+
+			// TODO: test SAS token
+			// accountSASSecret, err := createSecret(context.TODO(), azureCred, accountName+"-sas", accountSasToken)
+			// framework.ExpectNoError(err)
+
+			pod.Volumes[n].ContainerName = containerName
+			pod.Volumes[n].StorageAccountname = accountName
+			pod.Volumes[n].KeyVaultURL = *vault.Properties.VaultURI
+			pod.Volumes[n].KeyVaultSecretName = *accountKeySecret.Name
+			tpod, cleanup := pod.SetupWithPreProvisionedVolumes(client, namespace, t.CSIDriver)
+			// defer must be called here for resources not get removed before using them
+			for i := range cleanup {
+				defer cleanup[i]()
+			}
+
+			ginkgo.By("deploying the pod")
+			tpod.Create()
+			defer tpod.Cleanup()
+			ginkgo.By("checking that the pods command exits with no error")
+			tpod.WaitForSuccess()
+			ginkgo.By("sleep........")
+			time.Sleep(5 * time.Minute)
+		}
+	}
+}
+
+func createVault(ctx context.Context, cred azcore.TokenCredential) (*armkeyvault.Vault, error) {
+	vaultsClient, err := armkeyvault.NewVaultsClient(subscriptionID, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	pollerResp, err := vaultsClient.BeginCreateOrUpdate(
+		ctx,
+		resourceGroupName,
+		vaultName,
+		armkeyvault.VaultCreateOrUpdateParameters{
+			Location: to.Ptr(location),
+			Properties: &armkeyvault.VaultProperties{
+				SKU: &armkeyvault.SKU{
+					Family: to.Ptr(armkeyvault.SKUFamilyA),
+					Name:   to.Ptr(armkeyvault.SKUNameStandard),
+				},
+				TenantID: to.Ptr(TenantID),
+				AccessPolicies: []*armkeyvault.AccessPolicyEntry{
+					{
+						TenantID: to.Ptr(TenantID),
+						ObjectID: to.Ptr(ObjectID),
+						Permissions: &armkeyvault.Permissions{
+							Secrets: []*armkeyvault.SecretPermissions{
+								to.Ptr(armkeyvault.SecretPermissionsGet),
+							},
+						},
+					},
+				},
+			},
+		},
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := pollerResp.PollUntilDone(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &resp.Vault, nil
+}
+
+func cleanVault(ctx context.Context, cred azcore.TokenCredential) {
+	err := deleteVault(ctx, cred)
+	framework.ExpectNoError(err)
+
+	err = purgeDeleted(ctx, cred)
+	framework.ExpectNoError(err)
+}
+
+func deleteVault(ctx context.Context, cred azcore.TokenCredential) error {
+	vaultsClient, err := armkeyvault.NewVaultsClient(subscriptionID, cred, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = vaultsClient.Delete(ctx, resourceGroupName, vaultName, nil)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func purgeDeleted(ctx context.Context, cred azcore.TokenCredential) error {
+	vaultsClient, err := armkeyvault.NewVaultsClient(subscriptionID, cred, nil)
+	if err != nil {
+		return err
+	}
+
+	pollerResp, err := vaultsClient.BeginPurgeDeleted(ctx, vaultName, location, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = pollerResp.PollUntilDone(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createSecret(ctx context.Context, cred azcore.TokenCredential, secretName, secretValue string) (*armkeyvault.Secret, error) {
+	secretsClient, err := armkeyvault.NewSecretsClient(subscriptionID, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	secretResp, err := secretsClient.CreateOrUpdate(
+		ctx,
+		resourceGroupName,
+		vaultName,
+		secretName,
+		armkeyvault.SecretCreateOrUpdateParameters{
+			Properties: &armkeyvault.SecretProperties{
+				Attributes: &armkeyvault.SecretAttributes{
+					Enabled: to.Ptr(true),
+				},
+				Value: to.Ptr(secretValue),
+			},
+		},
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &secretResp.Secret, nil
+}

test/e2e/testsuites/pre_provisioned_keyvault_tester.go

@@ -52,6 +52,9 @@ type VolumeDetails struct {
 	DataSource         *DataSource
 	ContainerName      string
 	NodeStageSecretRef string
+	StorageAccountname string
+	KeyVaultURL        string
+	KeyVaultSecretName string
 }
 
 type VolumeMode int
@@ -204,6 +207,15 @@ func (volume *VolumeDetails) SetupPreProvisionedPersistentVolumeClaim(client cli
 	if volume.ContainerName != "" {
 		attrib["containerName"] = volume.ContainerName
 	}
+	if volume.StorageAccountname != "" {
+		attrib["storageAccountName"] = volume.StorageAccountname
+	}
+	if volume.KeyVaultURL != "" {
+		attrib["keyVaultURL"] = volume.KeyVaultURL
+	}
+	if volume.KeyVaultSecretName != "" {
+		attrib["keyVaultSecretName"] = volume.KeyVaultSecretName
+	}
 	nodeStageSecretRef := volume.NodeStageSecretRef
 	pv := csiDriver.GetPersistentVolume(volume.VolumeID, volume.FSType, volume.ClaimSize, volume.ReclaimPolicy, namespace.Name, attrib, nodeStageSecretRef)
 	tpv := NewTestPreProvisionedPersistentVolume(client, pv)