Add e2e test for OOM kill and Docker hung

Also fixes two minor bugs:

1. Change default Boskos wait timeout to 2 minutes.
This is because the current test timeout is configured to 10 minutes.
Running each test case taks 1-2 minutes, and each node will run 1-2 test
cases. 5 minutes timeout on waiting for Boskos may cause a test timeout,
which we want to avoid.

2. Create artifact subdir with 0755 rather than 0644.
Because execution bit should be set on the directories.

Author: Xuewei Zhang

test/e2e/metriconly/e2e_npd_test.go

@@ -49,7 +49,7 @@ var boskosProjectType = flag.String("boskos-project-type", "gce-project",
 	"specifies which project type to select from Boskos.")
 var boskosServerURL = flag.String("boskos-server-url", "http://boskos.test-pods.svc.cluster.local",
 	"specifies Boskos server URL.")
-var boskosWaitDuration = flag.Duration("boskos-wait-duration", 5*time.Minute,
+var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute,
 	"Duration to wait before quitting getting Boskos resource.")
 
 var computeService *compute.Service

test/e2e/metriconly/metrics_test.go

@@ -127,6 +127,32 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() {
 		})
 	})
 
+	ginkgo.Context("When OOM kills and docker hung happen", func() {
+
+		ginkgo.BeforeEach(func() {
+			err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120)
+			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err))
+			instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem OOMKill")
+			instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem DockerHung")
+		})
+
+		ginkgo.It("NPD should update problem_counter and problem_gauge", func() {
+			time.Sleep(5 * time.Second)
+			assertMetricValueInBound(instance,
+				"problem_counter", map[string]string{"reason": "DockerHung"},
+				1.0, 1.0)
+			assertMetricValueInBound(instance,
+				"problem_counter", map[string]string{"reason": "TaskHung"},
+				1.0, 1.0)
+			assertMetricValueInBound(instance,
+				"problem_gauge", map[string]string{"reason": "DockerHung", "type": "KernelDeadlock"},
+				1.0, 1.0)
+			assertMetricValueInBound(instance,
+				"problem_counter", map[string]string{"reason": "OOMKilling"},
+				1.0, 1.0)
+		})
+	})
+
 	ginkgo.AfterEach(func() {
 		defer func() {
 			err := instance.DeleteInstance()
@@ -139,7 +165,7 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() {
 			testSubdirName := strings.Replace(testText, " ", "_", -1)
 
 			artifactSubDir = path.Join(*artifactsDir, testSubdirName)
-			err := os.MkdirAll(artifactSubDir, os.ModeDir|0644)
+			err := os.MkdirAll(artifactSubDir, os.ModeDir|0755)
 			if err != nil {
 				fmt.Printf("Failed to create sub-directory to hold test artiface for test %s at %s\n",
 					testText, artifactSubDir)