Log PodExec stdout + stderr

e2e tests should log stdout / stderr of failed commands executed via
PodExec. "command terminated with exit code 1" is not really useful.

Author: jsafrane

test/e2e/storage/host_path_type.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"path"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/kubernetes/pkg/kubelet/events"
@@ -262,8 +262,9 @@ var _ = utils.SIGDescribe("HostPathType Character Device [Slow]", func() {
 		ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName))
 		targetCharDev = path.Join(hostBaseDir, "achardev")
 		ginkgo.By("Create a character device for further testing")
-		_, err := utils.PodExec(f, basePod, fmt.Sprintf("mknod %s c 89 1", path.Join(mountBaseDir, "achardev")))
-		framework.ExpectNoError(err)
+		cmd := fmt.Sprintf("mknod %s c 89 1", path.Join(mountBaseDir, "achardev"))
+		stdout, stderr, err := utils.PodExec(f, basePod, cmd)
+		framework.ExpectNoError(err, "command: %q, stdout: %s\nstderr: %s", cmd, stdout, stderr)
 	})
 
 	ginkgo.It("Should fail on mounting non-existent character device 'does-not-exist-char-dev' when HostPathType is HostPathCharDev", func() {
@@ -330,8 +331,9 @@ var _ = utils.SIGDescribe("HostPathType Block Device [Slow]", func() {
 		ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName))
 		targetBlockDev = path.Join(hostBaseDir, "ablkdev")
 		ginkgo.By("Create a block device for further testing")
-		_, err := utils.PodExec(f, basePod, fmt.Sprintf("mknod %s b 89 1", path.Join(mountBaseDir, "ablkdev")))
-		framework.ExpectNoError(err)
+		cmd := fmt.Sprintf("mknod %s b 89 1", path.Join(mountBaseDir, "ablkdev"))
+		stdout, stderr, err := utils.PodExec(f, basePod, cmd)
+		framework.ExpectNoError(err, "command %q: stdout: %s\nstderr: %s", cmd, stdout, stderr)
 	})
 
 	ginkgo.It("Should fail on mounting non-existent block device 'does-not-exist-blk-dev' when HostPathType is HostPathBlockDev", func() {

test/e2e/storage/persistent_volumes-local.go

@@ -1074,10 +1074,10 @@ func testReadFileContent(f *framework.Framework, testFileDir string, testFile st
 // Execute a read or write command in a pod.
 // Fail on error
 func podRWCmdExec(f *framework.Framework, pod *v1.Pod, cmd string) string {
-	out, err := utils.PodExec(f, pod, cmd)
-	framework.Logf("podRWCmdExec out: %q err: %v", out, err)
+	stdout, stderr, err := utils.PodExec(f, pod, cmd)
+	framework.Logf("podRWCmdExec cmd: %q, out: %q, stderr: %q, err: %v", cmd, stdout, stderr, err)
 	framework.ExpectNoError(err)
-	return out
+	return stdout
 }
 
 // Initialize test volume on node

test/e2e/storage/testsuites/snapshottable.go

@@ -276,8 +276,8 @@ func (s *snapshottableTestSuite) DefineTests(driver TestDriver, pattern testpatt
 				framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(cs, restoredPod.Name, restoredPod.Namespace))
 
 				command = "cat /mnt/test/data"
-				actualData, err := utils.PodExec(f, restoredPod, command)
-				framework.ExpectNoError(err)
+				actualData, stderr, err := utils.PodExec(f, restoredPod, command)
+				framework.ExpectNoError(err, "command %q: stdout: %s\nstderr: %s", command, actualData, stderr)
 				framework.ExpectEqual(actualData, originalMntTestData)
 
 				ginkgo.By("should delete the VolumeSnapshotContent according to its deletion policy")

test/e2e/storage/testsuites/volume_io.go

@@ -249,17 +249,19 @@ func writeToFile(f *framework.Framework, pod *v1.Pod, fpath, ddInput string, fsi
 	ginkgo.By(fmt.Sprintf("writing %d bytes to test file %s", fsize, fpath))
 	loopCnt := fsize / testpatterns.MinFileSize
 	writeCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do dd if=%s bs=%d >>%s 2>/dev/null; let i+=1; done", loopCnt, ddInput, testpatterns.MinFileSize, fpath)
-	_, err := utils.PodExec(f, pod, writeCmd)
-
+	stdout, stderr, err := utils.PodExec(f, pod, writeCmd)
+	if err != nil {
+		return fmt.Errorf("error writing to volume using %q: %s\nstdout: %s\nstderr: %s", writeCmd, err, stdout, stderr)
+	}
 	return err
 }
 
 // Verify that the test file is the expected size and contains the expected content.
 func verifyFile(f *framework.Framework, pod *v1.Pod, fpath string, expectSize int64, ddInput string) error {
 	ginkgo.By("verifying file size")
-	rtnstr, err := utils.PodExec(f, pod, fmt.Sprintf("stat -c %%s %s", fpath))
+	rtnstr, stderr, err := utils.PodExec(f, pod, fmt.Sprintf("stat -c %%s %s", fpath))
 	if err != nil || rtnstr == "" {
-		return fmt.Errorf("unable to get file size via `stat %s`: %v", fpath, err)
+		return fmt.Errorf("unable to get file size via `stat %s`: %v\nstdout: %s\nstderr: %s", fpath, err, rtnstr, stderr)
 	}
 	size, err := strconv.Atoi(strings.TrimSuffix(rtnstr, "\n"))
 	if err != nil {
@@ -270,9 +272,9 @@ func verifyFile(f *framework.Framework, pod *v1.Pod, fpath string, expectSize in
 	}
 
 	ginkgo.By("verifying file hash")
-	rtnstr, err = utils.PodExec(f, pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
+	rtnstr, stderr, err = utils.PodExec(f, pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
 	if err != nil {
-		return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err)
+		return fmt.Errorf("unable to test file hash via `md5sum %s`: %v\nstdout: %s\nstderr: %s", fpath, err, rtnstr, stderr)
 	}
 	actualHash := strings.TrimSuffix(rtnstr, "\n")
 	expectedHash, ok := md5hashes[expectSize]
@@ -291,10 +293,10 @@ func verifyFile(f *framework.Framework, pod *v1.Pod, fpath string, expectSize in
 // Delete `fpath` to save some disk space on host. Delete errors are logged but ignored.
 func deleteFile(f *framework.Framework, pod *v1.Pod, fpath string) {
 	ginkgo.By(fmt.Sprintf("deleting test file %s...", fpath))
-	_, err := utils.PodExec(f, pod, fmt.Sprintf("rm -f %s", fpath))
+	stdout, stderr, err := utils.PodExec(f, pod, fmt.Sprintf("rm -f %s", fpath))
 	if err != nil {
 		// keep going, the test dir will be deleted when the volume is unmounted
-		framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test", fpath, err)
+		framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test\nstdout: %s\nstderr: %s", fpath, err, stdout, stderr)
 	}
 }
 

test/e2e/storage/utils/utils.go

@@ -67,41 +67,41 @@ const (
 )
 
 // PodExec runs f.ExecCommandInContainerWithFullOutput to execute a shell cmd in target pod
-func PodExec(f *framework.Framework, pod *v1.Pod, shExec string) (string, error) {
-	stdout, _, err := f.ExecCommandInContainerWithFullOutput(pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", shExec)
-	return stdout, err
+func PodExec(f *framework.Framework, pod *v1.Pod, shExec string) (string, string, error) {
+	stdout, stderr, err := f.ExecCommandInContainerWithFullOutput(pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-c", shExec)
+	return stdout, stderr, err
 }
 
 // VerifyExecInPodSucceed verifies shell cmd in target pod succeed
 func VerifyExecInPodSucceed(f *framework.Framework, pod *v1.Pod, shExec string) {
-	_, err := PodExec(f, pod, shExec)
+	stdout, stderr, err := PodExec(f, pod, shExec)
 	if err != nil {
 		if exiterr, ok := err.(uexec.CodeExitError); ok {
 			exitCode := exiterr.ExitStatus()
 			framework.ExpectNoError(err,
-				"%q should succeed, but failed with exit code %d and error message %q",
-				shExec, exitCode, exiterr)
+				"%q should succeed, but failed with exit code %d and error message %q\nstdout: %s\nstderr: %s",
+				shExec, exitCode, exiterr, stdout, stderr)
 		} else {
 			framework.ExpectNoError(err,
-				"%q should succeed, but failed with error message %q",
-				shExec, err)
+				"%q should succeed, but failed with error message %q\nstdout: %s\nstderr: %s",
+				shExec, err, stdout, stderr)
 		}
 	}
 }
 
 // VerifyExecInPodFail verifies shell cmd in target pod fail with certain exit code
 func VerifyExecInPodFail(f *framework.Framework, pod *v1.Pod, shExec string, exitCode int) {
-	_, err := PodExec(f, pod, shExec)
+	stdout, stderr, err := PodExec(f, pod, shExec)
 	if err != nil {
 		if exiterr, ok := err.(clientexec.ExitError); ok {
 			actualExitCode := exiterr.ExitStatus()
 			framework.ExpectEqual(actualExitCode, exitCode,
-				"%q should fail with exit code %d, but failed with exit code %d and error message %q",
-				shExec, exitCode, actualExitCode, exiterr)
+				"%q should fail with exit code %d, but failed with exit code %d and error message %q\nstdout: %s\nstderr: %s",
+				shExec, exitCode, actualExitCode, exiterr, stdout, stderr)
 		} else {
 			framework.ExpectNoError(err,
-				"%q should fail with exit code %d, but failed with error message %q",
-				shExec, exitCode, err)
+				"%q should fail with exit code %d, but failed with error message %q\nstdout: %s\nstderr: %s",
+				shExec, exitCode, err, stdout, stderr)
 		}
 	}
 	framework.ExpectError(err, "%q should fail with exit code %d, but exit without error", shExec, exitCode)