refactor: Enable external triggering of graceful/ungraceful node reboots

This commit refactors the node reboot functionality in
`test/e2e/upgrade/monitor.go` to allow external triggering of graceful
and ungraceful node reboots.

The functionality was moved to `test/extended/util/nodes.go`,
and only used in the `Disrupt` function in
`test/e2e/upgrade/monitor.go`.

No new functionality was added, only the code was refactored.

This change enables other components to initiate node reboots.

Author: clobrano

test/e2e/upgrade/monitor.go

@@ -10,17 +10,15 @@ import (
 	"text/tabwriter"
 	"time"
 
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	configv1 "github.com/openshift/api/config/v1"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned"
+	"github.com/openshift/origin/test/extended/util"
 	"github.com/openshift/origin/test/extended/util/disruption"
-	"github.com/openshift/origin/test/extended/util/image"
 )
 
 type versionMonitor struct {
@@ -189,13 +187,13 @@ func (m *versionMonitor) Describe(f *framework.Framework) {
 }
 
 func (m *versionMonitor) Disrupt(ctx context.Context, kubeClient kubernetes.Interface, rebootPolicy string) {
-	rebootHard := false
+	rebootFunc := util.TriggerNodeRebootGraceful
 	switch rebootPolicy {
 	case "graceful":
 		framework.Logf("Periodically reboot master nodes with clean shutdown")
 	case "force":
 		framework.Logf("Periodically reboot master nodes without allowing for clean shutdown")
-		rebootHard = true
+		rebootFunc = util.TriggerNodeRebootUngraceful
 	case "":
 		return
 	}
@@ -212,7 +210,7 @@ func (m *versionMonitor) Disrupt(ctx context.Context, kubeClient kubernetes.Inte
 		rand.Shuffle(len(nodes.Items), func(i, j int) { nodes.Items[i], nodes.Items[j] = nodes.Items[j], nodes.Items[i] })
 		name := nodes.Items[0].Name
 		framework.Logf("DISRUPTION: Triggering reboot of %s", name)
-		if err := triggerReboot(kubeClient, name, 0, rebootHard); err != nil {
+		if err := rebootFunc(kubeClient, name); err != nil {
 			framework.Logf("Failed to reboot %s: %v", name, err)
 			continue
 		}
@@ -317,62 +315,3 @@ func versionString(update configv1.Update) string {
 		return "<empty>"
 	}
 }
-
-func triggerReboot(kubeClient kubernetes.Interface, target string, attempt int, rebootHard bool) error {
-	command := "echo 'reboot in 1 minute'; exec chroot /host shutdown -r 1"
-	if rebootHard {
-		command = "echo 'reboot in 1 minute'; exec chroot /host sudo systemd-run sh -c 'sleep 60 && reboot --force --force'"
-	}
-	isTrue := true
-	zero := int64(0)
-	name := fmt.Sprintf("reboot-%s-%d", target, attempt)
-	_, err := kubeClient.CoreV1().Pods("kube-system").Create(context.Background(), &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Annotations: map[string]string{
-				"test.openshift.io/upgrades-target": target,
-			},
-		},
-		Spec: corev1.PodSpec{
-			HostPID:       true,
-			RestartPolicy: corev1.RestartPolicyNever,
-			NodeName:      target,
-			Volumes: []corev1.Volume{
-				{
-					Name: "host",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/",
-						},
-					},
-				},
-			},
-			Containers: []corev1.Container{
-				{
-					Name: "reboot",
-					SecurityContext: &corev1.SecurityContext{
-						RunAsUser:  &zero,
-						Privileged: &isTrue,
-					},
-					Image: image.ShellImage(),
-					Command: []string{
-						"/bin/bash",
-						"-c",
-						command,
-					},
-					TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							MountPath: "/host",
-							Name:      "host",
-						},
-					},
-				},
-			},
-		},
-	}, metav1.CreateOptions{})
-	if errors.IsAlreadyExists(err) {
-		return triggerReboot(kubeClient, target, attempt+1, rebootHard)
-	}
-	return err
-}

test/extended/util/nodes.go

@@ -2,12 +2,16 @@ package util
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"time"
 
+	"github.com/openshift/origin/test/extended/util/image"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 )
 
 // GetClusterNodesByRole returns the cluster nodes by role
@@ -87,3 +91,70 @@ func DebugSelectedNodesRetryWithOptionsAndChroot(oc *CLI, selector string, debug
 func DebugAllNodesRetryWithOptionsAndChroot(oc *CLI, debugNodeNamespace string, cmd ...string) (map[string]string, error) {
 	return DebugSelectedNodesRetryWithOptionsAndChroot(oc, "", debugNodeNamespace, cmd...)
 }
+
+// TriggerNodeRebootGraceful initiates a graceful node reboot which allows the system to terminate processes cleanly before rebooting.
+func TriggerNodeRebootGraceful(kubeClient kubernetes.Interface, nodeName string) error {
+	command := "echo 'reboot in 1 minute'; exec chroot /host shutdown -r 1"
+	return triggerNodeReboot(kubeClient, nodeName, 0, command)
+}
+
+// TriggerNodeRebootUngraceful initiates an ungraceful node reboot which does not allow the system to terminate processes cleanly before rebooting.
+func TriggerNodeRebootUngraceful(kubeClient kubernetes.Interface, nodeName string) error {
+	command := "echo 'reboot in 1 minute'; exec chroot /host sudo systemd-run sh -c 'sleep 60 && reboot --force --force'"
+	return triggerNodeReboot(kubeClient, nodeName, 0, command)
+}
+
+func triggerNodeReboot(kubeClient kubernetes.Interface, nodeName string, attempt int, command string) error {
+	isTrue := true
+	zero := int64(0)
+	name := fmt.Sprintf("reboot-%s-%d", nodeName, attempt)
+	_, err := kubeClient.CoreV1().Pods("kube-system").Create(context.Background(), &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Annotations: map[string]string{
+				"test.openshift.io/disrupt-target": nodeName,
+			},
+		},
+		Spec: corev1.PodSpec{
+			HostPID:       true,
+			RestartPolicy: corev1.RestartPolicyNever,
+			NodeName:      nodeName,
+			Volumes: []corev1.Volume{
+				{
+					Name: "host",
+					VolumeSource: corev1.VolumeSource{
+						HostPath: &corev1.HostPathVolumeSource{
+							Path: "/",
+						},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name: "reboot",
+					SecurityContext: &corev1.SecurityContext{
+						RunAsUser:  &zero,
+						Privileged: &isTrue,
+					},
+					Image: image.ShellImage(),
+					Command: []string{
+						"/bin/bash",
+						"-c",
+						command,
+					},
+					TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							MountPath: "/host",
+							Name:      "host",
+						},
+					},
+				},
+			},
+		},
+	}, metav1.CreateOptions{})
+	if errors.IsAlreadyExists(err) {
+		return triggerNodeReboot(kubeClient, nodeName, attempt+1, command)
+	}
+	return err
+}