Merge pull request #579 from abhinavdahiya/testing_log_collector

Add log-collector to e2e tests

Author: Yifan Gu

e2e/log.go

@@ -0,0 +1,53 @@
+package e2e
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"fmt"
+
+	collector "github.com/coreos/ktestutil/log-collector"
+)
+
+var cr *collector.Collector
+
+func startLogging(keypath, dstDir string) error {
+	cr = collector.New(&collector.Config{
+		K8sClient:     client,
+		Namespace:     namespace,
+		RemoteKeyFile: keypath,
+	})
+	if err := cr.Start(); err != nil {
+		return fmt.Errorf("error starting log-collector: %v", err)
+	}
+
+	dirAbsPath, _ := filepath.Abs(dstDir)
+	os.Mkdir(dirAbsPath, 0777)
+	if err := cr.SetOutputToLocal(dirAbsPath); err != nil {
+		return fmt.Errorf("error starting log-collector: %v", err)
+	}
+	return nil
+}
+
+func collectLogs() error {
+	pods, err := cr.CollectPodLogs("*")
+	if err != nil {
+		return fmt.Errorf("error collecting logs from log-collector: %v", err)
+	}
+	log.Printf("[%d] log files collected for pods", len(pods))
+
+	services, err := cr.CollectServiceLogs("*")
+	if err != nil {
+		return fmt.Errorf("error collecting logs from log-collector: %v", err)
+	}
+	log.Printf("[%d] log files collected for services", len(services))
+	return nil
+}
+
+func cleanUpLogging() error {
+	if err := cr.Cleanup(); err != nil {
+		return fmt.Errorf("error cleaning-up log-collector: %v", err)
+	}
+	return nil
+}

e2e/main_test.go

@@ -30,6 +30,7 @@ func TestMain(m *testing.M) {
 	var kubeconfig = flag.String("kubeconfig", "../hack/quickstart/cluster/auth/kubeconfig", "absolute path to the kubeconfig file")
 	var keypath = flag.String("keypath", "", "path to private key for ssh client")
 	flag.IntVar(&expectedMasters, "expectedmasters", 1, "hint needed for certain tests to fail, skip, or block on missing resources")
+	var logOuputDir = flag.String("log-output-dir", "./logs", "logs from the tests will be populated in this directory")
 
 	flag.Parse()
 
@@ -59,9 +60,27 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
+	// start log-collector
+	// don't panic if it fails
+	if err := startLogging(*keypath, *logOuputDir); err != nil {
+		fmt.Println(err)
+	}
+
 	// run tests
 	exitCode := m.Run()
 
+	// cleanup log-collector assests
+	// don't panic if it fails
+	if err := cleanUpLogging(); err != nil {
+		fmt.Println(err)
+	}
+
+	// Collect all logs
+	// don't panic if it fails
+	if err := collectLogs(); err != nil {
+		fmt.Println(err)
+	}
+
 	if err := deleteNamespace(client, namespace); err != nil {
 		fmt.Println(err)
 		os.Exit(1)

glide.lock

@@ -49,3 +49,5 @@ import:
   - ssh
   - ssh/agent
   - curve25519
+- package: github.com/coreos/ktestutil
+  version: d598fab525dabcc7af2681ee0455efe47e055dd8 

glide.yaml

@@ -13,6 +13,8 @@ coreos:
         Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
           --volume var-lib-cni,kind=host,source=/var/lib/cni \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
+          --volume var-log,kind=host,source=/var/log \
+          --mount volume=var-log,target=/var/log \
           --mount volume=var-lib-cni,target=/var/lib/cni \
           --mount volume=opt-cni-bin,target=/opt/cni/bin"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin

hack/multi-node/user-data.sample

@@ -5,6 +5,7 @@ Environment="RKT_RUN_ARGS=\
 --uuid-file-save=/var/cache/kubelet-pod.uuid \
 --volume etc-resolv,kind=host,source=/etc/resolv.conf --mount volume=etc-resolv,target=/etc/resolv.conf \
 --volume opt-cni-bin,kind=host,source=/opt/cni/bin --mount volume=opt-cni-bin,target=/opt/cni/bin \
+--volume var-log,kind=host,source=/var/log --mount volume=var-log,target=/var/log \
 --volume var-lib-cni,kind=host,source=/var/lib/cni --mount volume=var-lib-cni,target=/var/lib/cni"
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests

hack/quickstart/kubelet.master

@@ -5,6 +5,7 @@ Environment="RKT_RUN_ARGS=\
 --uuid-file-save=/var/cache/kubelet-pod.uuid \
 --volume etc-resolv,kind=host,source=/etc/resolv.conf --mount volume=etc-resolv,target=/etc/resolv.conf \
 --volume opt-cni-bin,kind=host,source=/opt/cni/bin --mount volume=opt-cni-bin,target=/opt/cni/bin \
+--volume var-log,kind=host,source=/var/log --mount volume=var-log,target=/var/log \
 --volume var-lib-cni,kind=host,source=/var/lib/cni --mount volume=var-lib-cni,target=/var/lib/cni"
 EnvironmentFile=/etc/environment
 ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests

hack/quickstart/kubelet.worker

@@ -13,6 +13,8 @@ coreos:
         Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
           --volume var-lib-cni,kind=host,source=/var/lib/cni \
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
+          --volume var-log,kind=host,source=/var/log \
+          --mount volume=var-log,target=/var/log \
           --mount volume=var-lib-cni,target=/var/lib/cni \
           --mount volume=opt-cni-bin,target=/opt/cni/bin"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin