Merge pull request #273 from jiaxunsongucb/tests

test: change the way credentials are loaded

Author: k8s-ci-robot

Makefile

@@ -28,8 +28,8 @@ IMAGE_TAG ?= $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
 IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGE_NAME):latest
 BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 LDFLAGS ?= "-X ${PKG}/pkg/blob.driverVersion=${IMAGE_VERSION} -X ${PKG}/pkg/blob.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/blob.buildDate=${BUILD_DATE} -s -w -extldflags '-static'"
-E2E_HELM_OPTIONS ?= --set image.blob.pullPolicy=IfNotPresent --set image.blob.repository=$(REGISTRY)/$(IMAGE_NAME) --set image.blob.tag=$(IMAGE_VERSION)
-GINKGO_FLAGS = -ginkgo.noColor -ginkgo.v
+E2E_HELM_OPTIONS ?= --set image.blob.pullPolicy=Always --set image.blob.repository=$(REGISTRY)/$(IMAGE_NAME) --set image.blob.tag=$(IMAGE_VERSION)
+GINKGO_FLAGS = -ginkgo.v
 GO111MODULE = on
 GOPATH ?= $(shell go env GOPATH)
 GOBIN ?= $(GOPATH)/bin
@@ -79,15 +79,15 @@ e2e-teardown:
 
 .PHONY: blob
 blob:
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags ${LDFLAGS} -o _output/blobplugin ./pkg/blobplugin
+	CGO_ENABLED=0 GOOS=linux go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
 
 .PHONY: blob-windows
 blob-windows:
-	CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -o _output/blobplugin.exe ./pkg/blobplugin
+	CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin.exe ./pkg/blobplugin
 
 .PHONT: blob-darwin
 blob-darwin:
-	CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -o _output/blobplugin ./pkg/blobplugin
+	CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
 
 .PHONY: container
 container: blob
@@ -143,3 +143,7 @@ create-example-deployment:
 	kubectl apply -f deploy/example/storageclass-blobfuse.yaml
 	kubectl apply -f deploy/example/deployment.yaml
 	kubectl apply -f deploy/example/statefulset.yaml
+
+.PHONY: delete-metrics-svc
+delete-metrics-svc:
+	kubectl delete -f deploy/example/metrics/csi-blob-controller-svc.yaml --ignore-not-found

hack/verify-boilerplate.sh

@@ -30,7 +30,7 @@ REPO_ROOT=$(dirname "${BASH_SOURCE}")/..
 boilerDir="${REPO_ROOT}/hack/boilerplate"
 boiler="${boilerDir}/boilerplate.py"
 
-files_need_boilerplate=($(${boiler} --rootdir=${REPO_ROOT}))
+files_need_boilerplate=($(${boiler} --rootdir=${REPO_ROOT} --verbose))
 
 # Run boilerplate.py unit tests
 unitTestOut="$(mktemp)"

pkg/blobplugin/Dockerfile

@@ -18,7 +18,7 @@ ADD . .
 ARG TARGETARCH
 ARG TARGETOS
 ARG LDFLAGS
-RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "${LDFLAGS:--X sigs.k8s.io/blob-csi-driver/pkg/blob.driverVersion=latest}" -o _output/blobplugin ./pkg/blobplugin
+RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -a -ldflags "${LDFLAGS:--X sigs.k8s.io/blob-csi-driver/pkg/blob.driverVersion=latest}" -mod vendor -o _output/blobplugin ./pkg/blobplugin
 
 FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3
 COPY --from=builder /go/src/sigs.k8s.io/blob-csi-driver/_output/blobplugin /blobplugin

test/e2e/README.md

@@ -2,9 +2,33 @@
 
 ## Run E2E tests Locally
 ### Prerequisite
- - Make sure a kubernetes cluster(with version >= 1.13) is set up and kubeconfig is under `$HOME/.kube/config`
- - Copy out `/etc/kubernetes/azure.json` under one agent node to local machine
- > For AKS cluster, need to modify `resourceGroup` to the node resource group name inside `/etc/kubernetes/azure.json`
+ - Make sure a kubernetes cluster(with version >= 1.13) is set up and kubeconfig is under `$HOME/.kube/config`. Make sure kubectl is functional.
+ - Better to do: install Helm and Tiller 
+ ```
+# Use v2.11.0 helm to match tiller's version in clusters made by aks-engine
+curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | DESIRED_VERSION=v2.11.0 bash
+
+# Install Tiller for Helm version v2.11.0
+helm init --history-max 200 --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' --output yaml | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' | kubectl apply -f -
+```
+ - Set Azure credentials by environment variables
+ > You could get these variables from `/etc/kubernetes/azure.json` on a kubernetes cluster node
+```
+# Required environment variables:
+export set AZURE_TENANT_ID=
+export set AZURE_SUBSCRIPTION_ID=
+export set AZURE_CLIENT_ID=
+export set AZURE_CLIENT_SECRET=
+
+# Optional environment variables:
+# If the the test is not for the public Azure, e.g. Azure China Cloud, then you need to set AZURE_CLOUD_NAME and AZURE_LOCATION.
+# For Azure Stack Clound, you need to set AZURE_ENVIRONMENT_FILEPATH for your cloud environment.
+# If you have an existing resource group created for the test, then you need to set variable AZURE_RESOURCE_GROUP.
+export set AZURE_CLOUD_NAME=
+export set AZURE_LOCATION=
+export set AZURE_ENVIRONMENT_FILEPATH=
+export set AZURE_RESOURCE_GROUP=
+```
 
 ### Run test
 ```

test/e2e/pre_provisioning_test.go

@@ -72,7 +72,7 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Pre-Provisioned", func() {
 			}
 			_, err := blobDriver.DeleteVolume(context.Background(), req)
 			if err != nil {
-				ginkgo.Fail(fmt.Sprintf("create volume %q error: %v", volumeID, err))
+				ginkgo.Fail(fmt.Sprintf("delete volume %q error: %v", volumeID, err))
 			}
 		}
 	})

test/e2e/suite_test.go

@@ -71,14 +71,14 @@ var _ = ginkgo.BeforeSuite(func() {
 	handleFlags()
 	framework.AfterReadingAllFlags(&framework.TestContext)
 
-	if testutil.IsRunningInProw() {
-		creds, err := credentials.CreateAzureCredentialFile(false)
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		_, err = azureClient.EnsureResourceGroup(context.Background(), creds.ResourceGroup, creds.Location, nil)
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	creds, err := credentials.CreateAzureCredentialFile()
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	_, err = azureClient.EnsureResourceGroup(context.Background(), creds.ResourceGroup, creds.Location, nil)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
+	if testutil.IsRunningInProw() {
 		// Need to login to ACR using SP credential if we are running in Prow so we can push test images.
 		// If running locally, user should run 'docker login' before running E2E tests
 		registry := os.Getenv("REGISTRY")
@@ -89,23 +89,24 @@ var _ = ginkgo.BeforeSuite(func() {
 		err = cmd.Run()
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		log.Println("docker login is successful")
+	}
+
+	// Install Azure Blob Storage CSI driver on cluster from project root
+	e2eBootstrap := testCmd{
+		command:  "make",
+		args:     []string{"e2e-bootstrap"},
+		startLog: "Installing Azure Blob Storage CSI driver ...",
+		endLog:   "Azure Blob Storage CSI driver installed",
+	}
 
-		// Install Azure Blob Storage CSI driver on cluster from project root
-		e2eBootstrap := testCmd{
-			command:  "make",
-			args:     []string{"e2e-bootstrap"},
-			startLog: "Installing Azure Blob Storage CSI driver ...",
-			endLog:   "Azure Blob Storage CSI driver installed",
-		}
-
-		createMetricsSVC := testCmd{
-			command:  "make",
-			args:     []string{"create-metrics-svc"},
-			startLog: "create metrics service ...",
-			endLog:   "metrics service created",
-		}
-		execTestCmd([]testCmd{e2eBootstrap, createMetricsSVC})
+	createMetricsSVC := testCmd{
+		command:  "make",
+		args:     []string{"create-metrics-svc"},
+		startLog: "create metrics service ...",
+		endLog:   "metrics service created",
 	}
+	execTestCmd([]testCmd{e2eBootstrap, createMetricsSVC})
+
 	nodeid := os.Getenv("nodeid")
 	kubeconfig := os.Getenv(kubeconfigEnvVar)
 	blobDriver = blob.NewDriver(nodeid)
@@ -116,49 +117,53 @@ var _ = ginkgo.BeforeSuite(func() {
 })
 
 var _ = ginkgo.AfterSuite(func() {
-	if testutil.IsRunningInProw() {
-		createExampleDeployment := testCmd{
-			command:  "make",
-			args:     []string{"create-example-deployment"},
-			startLog: "create example deployments",
-			endLog:   "example deployments created",
-		}
-		execTestCmd([]testCmd{createExampleDeployment})
-		// sleep 120s waiting for deployment running complete
-		time.Sleep(120 * time.Second)
-
-		blobLog := testCmd{
-			command:  "bash",
-			args:     []string{"test/utils/blob_log.sh"},
-			startLog: "===================blob log===================",
-			endLog:   "==================================================",
-		}
-		e2eTeardown := testCmd{
-			command:  "make",
-			args:     []string{"e2e-teardown"},
-			startLog: "Uninstalling Azure Blob Storage CSI driver...",
-			endLog:   "Azure Blob Storage CSI driver uninstalled",
-		}
-		execTestCmd([]testCmd{blobLog, e2eTeardown})
-
-		// install/uninstall CSI Driver deployment scripts test
-		installDriver := testCmd{
-			command:  "bash",
-			args:     []string{"deploy/install-driver.sh", "master", "local"},
-			startLog: "===================install CSI Driver deployment scripts test===================",
-			endLog:   "===================================================",
-		}
-		uninstallDriver := testCmd{
-			command:  "bash",
-			args:     []string{"deploy/uninstall-driver.sh", "master", "local"},
-			startLog: "===================uninstall CSI Driver deployment scripts test===================",
-			endLog:   "===================================================",
-		}
-		execTestCmd([]testCmd{installDriver, uninstallDriver})
-
-		err := credentials.DeleteAzureCredentialFile()
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	createExampleDeployment := testCmd{
+		command:  "make",
+		args:     []string{"create-example-deployment"},
+		startLog: "create example deployments",
+		endLog:   "example deployments created",
+	}
+	execTestCmd([]testCmd{createExampleDeployment})
+	// sleep 120s waiting for deployment running complete
+	time.Sleep(120 * time.Second)
+
+	blobLog := testCmd{
+		command:  "bash",
+		args:     []string{"test/utils/blob_log.sh"},
+		startLog: "===================blob log===================",
+		endLog:   "==================================================",
+	}
+	deleteMetricsSVC := testCmd{
+		command:  "make",
+		args:     []string{"delete-metrics-svc"},
+		startLog: "delete metrics service...",
+		endLog:   "metrics service deleted",
 	}
+	e2eTeardown := testCmd{
+		command:  "make",
+		args:     []string{"e2e-teardown"},
+		startLog: "Uninstalling Azure Blob Storage CSI driver...",
+		endLog:   "Azure Blob Storage CSI driver uninstalled",
+	}
+	execTestCmd([]testCmd{blobLog, deleteMetricsSVC, e2eTeardown})
+
+	// install/uninstall CSI Driver deployment scripts test
+	installDriver := testCmd{
+		command:  "bash",
+		args:     []string{"deploy/install-driver.sh", "master", "local"},
+		startLog: "===================install CSI Driver deployment scripts test===================",
+		endLog:   "===================================================",
+	}
+	uninstallDriver := testCmd{
+		command:  "bash",
+		args:     []string{"deploy/uninstall-driver.sh", "master", "local"},
+		startLog: "===================uninstall CSI Driver deployment scripts test===================",
+		endLog:   "===================================================",
+	}
+	execTestCmd([]testCmd{installDriver, uninstallDriver})
+
+	err := credentials.DeleteAzureCredentialFile()
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 })
 
 func TestE2E(t *testing.T) {

test/integration/README.md

@@ -3,21 +3,32 @@ Integration test verifies the functionality of CSI driver as a standalone server
 
 ## Run Integration Tests Locally
 ### Prerequisite
- - make sure `GOPATH` is set and [csc](https://github.com/rexray/gocsi/tree/master/csc) tool is installed under `$GOPATH/bin/csc`
+ - Make sure `GOPATH` is set and [csc](https://github.com/rexray/gocsi/tree/master/csc) tool is installed under `$GOPATH/bin/csc`
 ```
-export set GOPATH=/root/go
+export set GOPATH=$HOME/go
 go get github.com/rexray/gocsi/csc
+cd $GOPATH/src/github.com/rexray/gocsi/csc
+make build
+export PATH=$PATH:$GOROOT/bin:$GOPATH/bin
 ```
 
- - set Azure credentials by environment variables
- > you could get these variables from `/etc/kubernetes/azure.json` on a kubernetes cluster node
+ - Set Azure credentials by environment variables
+ > You could get these variables from `/etc/kubernetes/azure.json` on a kubernetes cluster node
 ```
-export set tenantId=
-export set subscriptionId=
-export set aadClientId=
-export set aadClientSecret=
-export set resourceGroup=
-export set location=
+# Required environment variables:
+export set AZURE_TENANT_ID=
+export set AZURE_SUBSCRIPTION_ID=
+export set AZURE_CLIENT_ID=
+export set AZURE_CLIENT_SECRET=
+
+# Optional environment variables:
+# If the the test is not for the public Azure, e.g. Azure China Cloud, then you need to set AZURE_CLOUD_NAME and AZURE_LOCATION.
+# For Azure Stack Clound, you need to set AZURE_ENVIRONMENT_FILEPATH for your cloud environment.
+# If you have an existing resource group created for the test, then you need to set variable AZURE_RESOURCE_GROUP.
+export set AZURE_CLOUD_NAME=
+export set AZURE_LOCATION=
+export set AZURE_ENVIRONMENT_FILEPATH=
+export set AZURE_RESOURCE_GROUP=
 ```
 
 ### Run integration tests

test/integration/integration_test.go

@@ -26,45 +26,19 @@ import (
 
 	"sigs.k8s.io/blob-csi-driver/test/utils/azure"
 	"sigs.k8s.io/blob-csi-driver/test/utils/credentials"
-	"sigs.k8s.io/blob-csi-driver/test/utils/testutil"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestIntegrationOnAzurePublicCloud(t *testing.T) {
-	// Test on AzurePublicCloud
-	creds, err := credentials.CreateAzureCredentialFile(false)
+func TestIntegration(t *testing.T) {
+	creds, err := credentials.CreateAzureCredentialFile()
 	defer func() {
 		err := credentials.DeleteAzureCredentialFile()
 		assert.NoError(t, err)
 	}()
 	assert.NoError(t, err)
 	assert.NotNil(t, creds)
 
-	testIntegration(t, creds)
-}
-
-func TestIntegrationOnAzureChinaCloud(t *testing.T) {
-	if testutil.IsRunningInProw() {
-		t.Skipf("Skipping integration test on Azure China Cloud because Prow only tests on Azure Public Cloud at the moment")
-	}
-
-	// Test on AzureChinaCloud
-	creds, err := credentials.CreateAzureCredentialFile(true)
-	defer func() {
-		err := credentials.DeleteAzureCredentialFile()
-		assert.NoError(t, err)
-	}()
-
-	if err != nil {
-		// Skip the test if Azure China Cloud credentials are not supplied
-		t.Skipf("Skipping integration test on Azure China Cloud due to the following error %v", err)
-	}
-	assert.NotNil(t, creds)
-	testIntegration(t, creds)
-}
-
-func testIntegration(t *testing.T, creds *credentials.Credentials) {
 	os.Setenv("AZURE_CREDENTIAL_FILE", credentials.TempAzureCredentialFilePath)
 
 	azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)

test/sanity/README.md

@@ -1,14 +1,31 @@
 ## Sanity Tests
-Testing the Azure File CSI driver using the [`sanity`](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) package test suite.
+Testing the Azure Blob CSI driver using the [`sanity`](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) package test suite.
 
 ## Run Integration Tests Locally
 ### Prerequisite
- - make sure `GOPATH` is set
+ - Make sure `GOPATH` is set
 
- - set the environment variable AZURE_CREDENTIAL_FILE with the path to cloud provider config file only if you have the file at a different location than `/etc/kubernetes/azure.json`
- > By default Cloud provider config file is present at `/etc/kubernetes/azure.json` on a kubernetes cluster node
 ```
-export set AZURE_CREDENTIAL_FILE=
+export set GOPATH=$HOME/go
+```
+
+ - Set Azure credentials by environment variables
+ > You could get these variables from `/etc/kubernetes/azure.json` on a kubernetes cluster node
+```
+# Required environment variables:
+export set AZURE_TENANT_ID=
+export set AZURE_SUBSCRIPTION_ID=
+export set AZURE_CLIENT_ID=
+export set AZURE_CLIENT_SECRET=
+
+# Optional environment variables:
+# If the the test is not for the public Azure, e.g. Azure China Cloud, then you need to set AZURE_CLOUD_NAME and AZURE_LOCATION.
+# For Azure Stack Clound, you need to set AZURE_ENVIRONMENT_FILEPATH for your cloud environment.
+# If you have an existing resource group created for the test, then you need to set variable AZURE_RESOURCE_GROUP.
+export set AZURE_CLOUD_NAME=
+export set AZURE_LOCATION=
+export set AZURE_ENVIRONMENT_FILEPATH=
+export set AZURE_RESOURCE_GROUP=
 ```
 
 ### Run integration tests

test/sanity/run-test.sh

@@ -36,4 +36,3 @@ _output/blobplugin --endpoint "$endpoint" --nodeid "$nodeid" -v=5 &
 echo "Begin to run sanity test..."
 readonly CSI_SANITY_BIN='csi-test/cmd/csi-sanity/csi-sanity'
 "$CSI_SANITY_BIN" --ginkgo.v --csi.endpoint=$endpoint -ginkgo.skip="should fail when requesting to create a volume with already existing name and different capacity"
-