Dump driver logs to test artifacts

Author: msau42

test/k8s-integration/cluster.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 
 	apimachineryversion "k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
 )
 
@@ -43,7 +45,7 @@ func clusterDownGKE(gceZone, gceRegion string) error {
 		return err
 	}
 
-	cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
+	cmd := exec.Command("gcloud", "container", "clusters", "delete", *gkeTestClusterName,
 		locationArg, locationVal, "--quiet")
 	err = runCommand("Bringing Down E2E Cluster on GKE", cmd)
 	if err != nil {
@@ -140,21 +142,21 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, use
 	}
 
 	out, err := exec.Command("gcloud", "container", "clusters", "list", locationArg, locationVal,
-		"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
+		"--filter", fmt.Sprintf("name=%s", *gkeTestClusterName)).CombinedOutput()
 
 	if err != nil {
 		return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
 	}
 	if len(out) > 0 {
-		klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
+		klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", *gkeTestClusterName)
 		err = clusterDownGKE(gceZone, gceRegion)
 		if err != nil {
 			return err
 		}
 	}
 
 	var cmd *exec.Cmd
-	cmdParams := []string{"container", "clusters", "create", gkeTestClusterName,
+	cmdParams := []string{"container", "clusters", "create", *gkeTestClusterName,
 		locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes),
 		"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType}
 	if isVariableSet(gkeClusterVer) {
@@ -276,7 +278,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
 		"--deployment=gke",
 		fmt.Sprintf("--gcp-node-image=%s", imageType),
 		"--gcp-network=default",
-		fmt.Sprintf("--cluster=%s", gkeTestClusterName),
+		fmt.Sprintf("--cluster=%s", *gkeTestClusterName),
 		fmt.Sprintf("--gke-environment=%s", gkeEnv),
 		fmt.Sprintf("%s=%s", locationArg, locationVal),
 		fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
@@ -336,3 +338,36 @@ func mustGetKubeClusterVersion() string {
 	}
 	return ver
 }
+
+// getKubeConfig returns the full path to the
+// kubeconfig file set in $KUBECONFIG env.
+// If unset, then it defaults to $HOME/.kube/config
+func getKubeConfig() (string, error) {
+	config, ok := os.LookupEnv("KUBECONFIG")
+	if ok {
+		return config, nil
+	}
+	homeDir, ok := os.LookupEnv("HOME")
+	if !ok {
+		return "", fmt.Errorf("HOME env not set")
+	}
+	return filepath.Join(homeDir, ".kube/config"), nil
+}
+
+// getKubeClient returns a Kubernetes client interface
+// for the test cluster
+func getKubeClient() (kubernetes.Interface, error) {
+	kubeConfig, err := getKubeConfig()
+	if err != nil {
+		return nil, err
+	}
+	config, err := clientcmd.BuildConfigFromFlags("", kubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create config: %v", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create client: %v", err)
+	}
+	return kubeClient, nil
+}

test/k8s-integration/driver.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
+
+	"k8s.io/kubernetes/test/e2e/framework/podlogs"
 )
 
 func getOverlayDir(pkgDir, deployOverlayName string) string {
@@ -65,7 +68,7 @@ func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayNa
 
 	// TODO (#139): wait for driver to be running
 	time.Sleep(time.Minute)
-	statusCmd := exec.Command("kubectl", "describe", "pods", "-n", "default")
+	statusCmd := exec.Command("kubectl", "describe", "pods", "-n", driverNamespace)
 	err = runCommand("Checking driver pods", statusCmd)
 	if err != nil {
 		return fmt.Errorf("failed to check driver pods: %v", err)
@@ -121,3 +124,29 @@ func deleteImage(stagingImage, stagingVersion string) error {
 	}
 	return nil
 }
+
+// dumpDriverLogs will watch all pods in the driver namespace
+// and copy its logs to the test artifacts directory, if set.
+// It returns a context.CancelFunc that needs to be invoked when
+// the test is finished.
+func dumpDriverLogs() (context.CancelFunc, error) {
+	// Dump all driver logs to the test artifacts
+	artifactsDir, ok := os.LookupEnv("ARTIFACTS")
+	if ok {
+		client, err := getKubeClient()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get kubeclient: %v", err)
+		}
+		out := podlogs.LogOutput{
+			StatusWriter:  os.Stdout,
+			LogPathPrefix: filepath.Join(artifactsDir, "pd-csi-driver") + "/",
+		}
+		ctx, cancel := context.WithCancel(context.Background())
+		if err = podlogs.CopyAllLogs(ctx, client, driverNamespace, out); err != nil {
+			return cancel, fmt.Errorf("failed to start pod logger: %v", err)
+		}
+		return cancel, nil
+	}
+	return nil, nil
+
+}

test/k8s-integration/main.go

@@ -25,27 +25,29 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/klog"
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
 )
 
 var (
 	// Kubernetes cluster flags
-	teardownCluster   = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
-	teardownDriver    = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
-	bringupCluster    = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
-	platform          = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
-	gceZone           = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
-	gceRegion         = flag.String("gce-region", "", "region that gke regional cluster should be created in")
-	kubeVersion       = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
-	testVersion       = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
-	kubeFeatureGates  = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
-	localK8sDir       = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
-	deploymentStrat   = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
-	gkeClusterVer     = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
-	numNodes          = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
-	imageType         = flag.String("image-type", "cos", "the image type to use for the cluster")
-	gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
+	teardownCluster    = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
+	teardownDriver     = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
+	bringupCluster     = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	platform           = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
+	gceZone            = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
+	gceRegion          = flag.String("gce-region", "", "region that gke regional cluster should be created in")
+	kubeVersion        = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
+	testVersion        = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
+	kubeFeatureGates   = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
+	localK8sDir        = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
+	deploymentStrat    = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
+	gkeClusterVer      = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
+	numNodes           = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
+	imageType          = flag.String("image-type", "cos", "the image type to use for the cluster")
+	gkeReleaseChannel  = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
+	gkeTestClusterName = flag.String("gke-cluster-name", "gcp-pd-csi-driver-test-cluster", "GKE cluster name")
 
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
@@ -69,7 +71,7 @@ const (
 	pdImagePlaceholder        = "gke.gcr.io/gcp-compute-persistent-disk-csi-driver"
 	k8sInDockerBuildBinDir    = "_output/dockerized/bin/linux/amd64"
 	k8sOutOfDockerBuildBinDir = "_output/bin"
-	gkeTestClusterName        = "gcp-pd-csi-driver-test-cluster"
+	driverNamespace           = "gce-pd-csi-driver"
 )
 
 func init() {
@@ -79,7 +81,7 @@ func init() {
 func main() {
 	flag.Parse()
 
-	if !*inProw && !*useGKEManagedDriver {
+	if !*inProw && *doDriverBuild {
 		ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
 	}
 
@@ -302,7 +304,6 @@ func handle() error {
 		err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
 		if *teardownDriver {
 			defer func() {
-				// TODO (#140): collect driver logs
 				if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
 					klog.Errorf("failed to delete driver: %v", teardownErr)
 				}
@@ -311,6 +312,17 @@ func handle() error {
 		if err != nil {
 			return fmt.Errorf("failed to install CSI Driver: %v", err)
 		}
+
+		// Dump all driver logs to the test artifacts
+		cancel, err := dumpDriverLogs()
+		if err != nil {
+			return fmt.Errorf("failed to start driver logging: %v", err)
+		}
+		defer func() {
+			if cancel != nil {
+				cancel()
+			}
+		}()
 	}
 
 	var cloudProviderArgs []string
@@ -423,8 +435,11 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 		return err
 	}
 
-	homeDir, _ := os.LookupEnv("HOME")
-	os.Setenv("KUBECONFIG", filepath.Join(homeDir, ".kube/config"))
+	kubeconfig, err := getKubeConfig()
+	if err != nil {
+		return err
+	}
+	os.Setenv("KUBECONFIG", kubeconfig)
 
 	artifactsDir, ok := os.LookupEnv("ARTIFACTS")
 	reportArg := ""