Merge pull request #559 from jingxu97/July/test

update driver test code for Windows

Author: k8s-ci-robot

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -4,6 +4,13 @@ kind: ServiceAccount
 metadata:
   name: csi-gce-pd-node-sa
 
+---
+##### Node Service Account, Roles, RoleBindings
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-gce-pd-node-sa-win
+
 ---
 ##### Controller Service Account, Roles, Rolebindings
 apiVersion: v1
@@ -195,7 +202,7 @@ roleRef:
   name: csi-gce-pd-node-deploy-win
 subjects:
 - kind: ServiceAccount
-  name: csi-gce-pd-node-sa
+  name: csi-gce-pd-node-sa-win
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

deploy/kubernetes/images/alpha/image.yaml

@@ -15,4 +15,4 @@ metadata:
 imageTag:
   name: gke.gcr.io/k8s-staging-csi/csi-node-driver-registrar-win
   newName: gcr.io/k8s-staging-csi/csi-node-driver-registrar
-  newTag: "amd64-windows-v20200428-v1.3.0-26-g510710d5"
+  newTag: "amd64-windows-canary"

test/k8s-integration/driver-config.go

@@ -45,6 +45,7 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
 
 	// Fill in template parameters. Capabilities can be found here:
 	// https://github.com/kubernetes/kubernetes/blob/b717be8269a4f381ab6c23e711e8924bc1f64c93/test/e2e/storage/testsuites/testdriver.go#L136
+
 	caps := []string{
 		"persistence",
 		"block",
@@ -56,6 +57,12 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
 	var fsTypes []string
 	if platform == "windows" {
 		fsTypes = []string{"ntfs"}
+		caps = []string{
+			"persistence",
+			"exec",
+			"multipods",
+			"topology",
+		}
 	} else {
 		fsTypes = []string{
 			"ext2",

test/k8s-integration/driver.go

@@ -8,16 +8,18 @@ import (
 	"path/filepath"
 	"time"
 
+	"k8s.io/klog"
 	"k8s.io/kubernetes/test/e2e/framework/podlogs"
 )
 
 func getOverlayDir(pkgDir, deployOverlayName string) string {
 	return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName)
 }
 
-func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error {
+func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error {
 	if doDriverBuild {
 		// Install kustomize
+		klog.Infof("Installing kustomize")
 		out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
 		if err != nil {
 			return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err)
@@ -41,6 +43,17 @@ func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayNa
 		if err != nil {
 			return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err)
 		}
+		if platform == "windows" {
+			out, err = exec.Command(
+				filepath.Join(pkgDir, "bin", "kustomize"),
+				"edit",
+				"set",
+				"image",
+				fmt.Sprintf("%s-win=%s-win:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err)
+			}
+		}
 	}
 
 	// setup service account file for secret creation
@@ -65,11 +78,18 @@ func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayNa
 	if err != nil {
 		return fmt.Errorf("failed to deploy driver: %v", err)
 	}
-
+	klog.Infof("Deploying driver")
 	// TODO (#139): wait for driver to be running
-	time.Sleep(time.Minute)
-	statusCmd := exec.Command("kubectl", "describe", "pods", "-n", driverNamespace)
-	err = runCommand("Checking driver pods", statusCmd)
+	if platform == "windows" {
+		klog.Infof("Waiting 15 minutes for the driver to start on Windows")
+		time.Sleep(15 * time.Minute)
+	} else {
+		klog.Infof("Waiting 5 minutes for the driver to start on Linux")
+		time.Sleep(5 * time.Minute)
+	}
+	out, err = exec.Command("kubectl", "describe", "pods", "-n", driverNamespace).CombinedOutput()
+	klog.Infof("describe pods \n %s", string(out))
+
 	if err != nil {
 		return fmt.Errorf("failed to describe pods: %v", err)
 	}
@@ -99,18 +119,22 @@ func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
 		return err
 	}
 	var cmd *exec.Cmd
-	if platform != "windows" {
-		cmd = exec.Command("make", "-C", pkgDir, "push-container",
-			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
-			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
-	} else {
-		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909",
-			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
-			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
-	}
-	err = runCommand("Pushing GCP Container", cmd)
+
+	cmd = exec.Command("make", "-C", pkgDir, "push-container",
+		fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+		fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	err = runCommand("Pushing GCP Container for Linux", cmd)
 	if err != nil {
-		return fmt.Errorf("failed to run make command: err: %v", err)
+		return fmt.Errorf("failed to run make command for linux: err: %v", err)
+	}
+	if platform == "windows" {
+		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-ltsc2019",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s-win", stagingImage))
+		err = runCommand("Building and Pushing GCP Container for Windows", cmd)
+		if err != nil {
+			return fmt.Errorf("failed to run make command for windows: err: %v", err)
+		}
 	}
 	return nil
 }

test/k8s-integration/main.go

@@ -79,6 +79,8 @@ func init() {
 }
 
 func main() {
+	klog.InitFlags(nil)
+	flag.Set("logtostderr", "true")
 	flag.Parse()
 
 	if !*inProw && *doDriverBuild {
@@ -298,27 +300,41 @@ func handle() error {
 			}
 		}()
 	}
+
 	// For windows cluster, when cluster is up, all Windows nodes are tainted with NoSchedule to avoid linux pods
 	// being scheduled to Windows nodes. When running windows tests, we need to remove the taint.
 	if *platform == "windows" {
-		nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "beta.kubernetes.io/os=windows", "-o", "name")
+		nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=windows", "-o", "name")
 		out, err := nodesCmd.CombinedOutput()
 		if err != nil {
-			return fmt.Errorf("failed to get nodes: %v", err)
+			return fmt.Errorf("failed to get windows nodes: %v", err)
 		}
 		nodes := strings.Fields(string(out))
 		for _, node := range nodes {
 			taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os:NoSchedule-")
-			_, err = taintCmd.CombinedOutput()
+			out, err := taintCmd.CombinedOutput()
 			if err != nil {
 				return fmt.Errorf("failed to untaint windows node %v", err)
 			}
+			klog.Infof("untaint windows nodes: %s, output %s", node, string(out))
+		}
+
+		// It typically takes 5+ minutes to download Windows container image. To avoid tests being timed out,
+		// pre-pulling the test images as best effort.
+		klog.Infof("Prepulling test images.")
+		err = os.Setenv("PREPULL_IMAGE", filepath.Join(pkgDir, "test", "k8s-integration", "prepull.yaml"))
+		if err != nil {
+			return err
+		}
+		out, err = exec.Command(filepath.Join(pkgDir, "test", "k8s-integration", "prepull-image.sh")).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to prepull images: %s, err: %v", out, err)
 		}
 	}
 
 	if !*useGKEManagedDriver {
 		// Install the driver and defer its teardown
-		err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
+		err := installDriver(*platform, goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
 		if *teardownDriver {
 			defer func() {
 				if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
@@ -342,6 +358,30 @@ func handle() error {
 		}()
 	}
 
+	// For windows cluster, it has both Windows nodes and Linux nodes. Before triggering the tests, taint Linux nodes
+	// with NoSchedule to avoid test pods being scheduled on Linux. Need to do this step after driver is deployed.
+	// Also the test framework will not proceed to run tests unless all nodes are ready
+	// AND schedulable. Allow not-ready nodes since we make Linux nodes
+	// unschedulable.
+	allowedNotReadyNodes := 0
+	if *platform == "windows" {
+		nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=linux", "-o", "name")
+		out, err := nodesCmd.CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to get linux nodes: %v", err)
+		}
+		nodes := strings.Fields(string(out))
+		allowedNotReadyNodes = len(nodes)
+		for _, node := range nodes {
+			taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os:NoSchedule")
+			out, err := taintCmd.CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("failed to untaint windows node %v", err)
+			}
+			klog.Infof("taint linux nodes: %s, output %s", node, string(out))
+		}
+	}
+
 	var cloudProviderArgs []string
 	var err error
 	switch *deploymentStrat {
@@ -374,7 +414,8 @@ func handle() error {
 		for _, scFile := range storageClasses {
 			outputDir := strings.TrimSuffix(scFile, ".yaml")
 			testOutputDirs = append(testOutputDirs, outputDir)
-			if err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, scFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat, outputDir); err != nil {
+			if err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, scFile,
+				*snapshotClassFile, cloudProviderArgs, *deploymentStrat, outputDir, allowedNotReadyNodes); err != nil {
 				ginkgoErrors = append(ginkgoErrors, err.Error())
 			}
 		}
@@ -455,19 +496,20 @@ func setEnvProject(project string) error {
 }
 
 func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProviderArgs []string) error {
-	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs, "")
+	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs, "", 0)
 }
 
-func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat, reportPrefix string) error {
+func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string,
+	cloudProviderArgs []string, deploymentStrat, reportPrefix string, allowedNotReadyNodes int) error {
 	testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
 	if err != nil {
 		return err
 	}
 	testConfigArg := fmt.Sprintf("--storage.testdriver=%s", testDriverConfigFile)
-	return runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg, cloudProviderArgs, reportPrefix)
+	return runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg, cloudProviderArgs, reportPrefix, allowedNotReadyNodes)
 }
 
-func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, cloudProviderArgs []string, reportPrefix string) error {
+func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, cloudProviderArgs []string, reportPrefix string, allowedNotReadyNodes int) error {
 	err := os.Chdir(testDir)
 	if err != nil {
 		return err
@@ -494,7 +536,7 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 	}
 	ginkgoArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s", testFocus, testSkip)
 	if *platform == "windows" {
-		ginkgoArgs = ginkgoArgs + fmt.Sprintf(" --node-os-distro=%s", *platform)
+		ginkgoArgs = ginkgoArgs + fmt.Sprintf(" --node-os-distro=%s --allowed-not-ready-nodes=%d", *platform, allowedNotReadyNodes)
 	}
 	testArgs := fmt.Sprintf("%s %s %s",
 		ginkgoArgs,

test/k8s-integration/prepull-image.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+if [[ -z "${PREPULL_IMAGE}" ]]; then
+    # Pre-pull all the test images. The images are currently hard-coded.
+    # Eventually, we should get the list directly from
+    # https://github.com/kubernetes-sigs/windows-testing/blob/master/images/PullImages.ps1
+    curl https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull-1.18.yaml -o prepull.yaml
+    PREPULL_IMAGE=prepull.yaml
+    echo ${PREPULL_IMAGE}
+fi
+
+kubectl create -f ${PREPULL_IMAGE}
+# Wait 10 minutes for the test images to be pulled onto the nodes.
+sleep 15m
+echo "sleep 15m"
+# Check the status of the pods.
+kubectl get pods -o wide
+# Delete the pods anyway since pre-pulling is best-effort
+kubectl delete -f ${PREPULL_IMAGE}
+# Wait a few more minutes for the pod to be cleaned up.
+sleep 5m