Merge pull request #29986 from clobrano/tnf-ungraceful-shutdown

OCPEDGE-1755: Add TNF E2E tests for ungraceful node shutdown

Author: openshift-merge-bot[bot]

test/e2e/upgrade/monitor.go

@@ -10,17 +10,15 @@ import (
 	"text/tabwriter"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	configv1 "github.com/openshift/api/config/v1"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned"
+	"github.com/openshift/origin/test/extended/util"
 	"github.com/openshift/origin/test/extended/util/disruption"
-	"github.com/openshift/origin/test/extended/util/image"
 )
 
 type versionMonitor struct {
@@ -189,13 +187,13 @@ func (m *versionMonitor) Describe(f *framework.Framework) {
 }
 
 func (m *versionMonitor) Disrupt(ctx context.Context, kubeClient kubernetes.Interface, rebootPolicy string) {
-	rebootHard := false
+	rebootFunc := util.TriggerNodeRebootGraceful
 	switch rebootPolicy {
 	case "graceful":
 		framework.Logf("Periodically reboot master nodes with clean shutdown")
 	case "force":
 		framework.Logf("Periodically reboot master nodes without allowing for clean shutdown")
-		rebootHard = true
+		rebootFunc = util.TriggerNodeRebootUngraceful
 	case "":
 		return
 	}
@@ -212,7 +210,7 @@ func (m *versionMonitor) Disrupt(ctx context.Context, kubeClient kubernetes.Inte
 		rand.Shuffle(len(nodes.Items), func(i, j int) { nodes.Items[i], nodes.Items[j] = nodes.Items[j], nodes.Items[i] })
 		name := nodes.Items[0].Name
 		framework.Logf("DISRUPTION: Triggering reboot of %s", name)
-		if err := triggerReboot(kubeClient, name, 0, rebootHard); err != nil {
+		if err := rebootFunc(kubeClient, name); err != nil {
 			framework.Logf("Failed to reboot %s: %v", name, err)
 			continue
 		}
@@ -317,62 +315,3 @@ func versionString(update configv1.Update) string {
 		return "<empty>"
 	}
 }
-
-func triggerReboot(kubeClient kubernetes.Interface, target string, attempt int, rebootHard bool) error {
-	command := "echo 'reboot in 1 minute'; exec chroot /host shutdown -r 1"
-	if rebootHard {
-		command = "echo 'reboot in 1 minute'; exec chroot /host sudo systemd-run sh -c 'sleep 60 && reboot --force --force'"
-	}
-	isTrue := true
-	zero := int64(0)
-	name := fmt.Sprintf("reboot-%s-%d", target, attempt)
-	_, err := kubeClient.CoreV1().Pods("kube-system").Create(context.Background(), &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Annotations: map[string]string{
-				"test.openshift.io/upgrades-target": target,
-			},
-		},
-		Spec: corev1.PodSpec{
-			HostPID:       true,
-			RestartPolicy: corev1.RestartPolicyNever,
-			NodeName:      target,
-			Volumes: []corev1.Volume{
-				{
-					Name: "host",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/",
-						},
-					},
-				},
-			},
-			Containers: []corev1.Container{
-				{
-					Name: "reboot",
-					SecurityContext: &corev1.SecurityContext{
-						RunAsUser:  &zero,
-						Privileged: &isTrue,
-					},
-					Image: image.ShellImage(),
-					Command: []string{
-						"/bin/bash",
-						"-c",
-						command,
-					},
-					TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							MountPath: "/host",
-							Name:      "host",
-						},
-					},
-				},
-			},
-		},
-	}, metav1.CreateOptions{})
-	if errors.IsAlreadyExists(err) {
-		return triggerReboot(kubeClient, target, attempt+1, rebootHard)
-	}
-	return err
-}