🐛 Script to detect invalid WWN/RAID configuration of hbmh. (#1164)

🐛 Detect invalid WWN configuration of hbmh.

Author: guettli

api/v1beta1/conditions_const.go

@@ -177,6 +177,10 @@ const (
 	CloudInitNotInstalledReason = "CloudInitNotInstalled"
 	// ServerNotFoundReason indicates that a bare metal server could not be found.
 	ServerNotFoundReason = "ServerNotFound"
+	// LinuxOnOtherDiskFoundReason indicates that the server can't be provisioned on the given WWN, since the reboot would fail.
+	LinuxOnOtherDiskFoundReason = "LinuxOnOtherDiskFound"
+	// SSHToRescueSystemFailedReason indicates that the rescue system can't be reached via ssh.
+	SSHToRescueSystemFailedReason = "SSHToRescueSystemFailed"
 )
 
 const (

controllers/hetznerbaremetalhost_controller_test.go

@@ -891,4 +891,5 @@ name="eth0" model="Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express
 	sshClient.On("ExecuteInstallImage", mock.Anything).Return(sshclient.Output{})
 	sshClient.On("Reboot").Return(sshclient.Output{})
 	sshClient.On("GetCloudInitOutput").Return(sshclient.Output{StdOut: "dummy content of /var/log/cloud-init-output.log"})
+	sshClient.On("DetectLinuxOnAnotherDisk", mock.Anything).Return(sshclient.Output{})
 }

hack/filter-caph-controller-manager-logs.py

@@ -20,7 +20,9 @@
 
 keys_to_skip = ['controller', 'controllerGroup', 'controllerKind', 'reconcileID',
                 'HetznerCluster', 'Cluster',
-                'namespace', 'name', 'Machine', 'stack', 'stacktrace']
+                'namespace', 'name', 'Machine', 'stack', 'stacktrace',
+                'logger',
+                ]
 
 rows_to_skip = [
     'controller-runtime.webhook', 'certwatcher/certwatcher', 'Registering a validating webhook',

pkg/services/baremetal/client/mocks/ssh/Client.go

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+trap 'echo "Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
+
+function usage() {
+    echo "$0 wwn1 [wwn2 ...]"
+    echo "    Check if there is a Linux partition, but skip all WWNs given as arguments"
+    echo "    Background: If we provision a disk, then there must not be a Linux OS on an other partition"
+    echo "    otherwise it is likely that the old OS gets booted, and not the new OS."
+    echo "    Exit 0: If there is no Linux installation found."
+    echo "    Exit 1: There is a Linux on a different disk.".
+    echo "    Exit 3: Unexpected error."
+    echo "Existing WWNs:"
+    lsblk -oNAME,WWN | grep -vi loop || true
+}
+
+if [ $# -eq 0 ]; then
+    echo "Error: No WWN was provided."
+    echo
+    usage
+    exit 3
+fi
+
+# Iterate over all input arguments
+for wwn in "$@"; do
+    if ! lsblk -l -oWWN | grep -qP '^'${wwn}'$'; then
+        echo "$wwn is not a WWN of this machine"
+        echo
+        usage
+        exit 3
+    fi
+done
+fail=0
+while read name wwn type parttype; do
+    if [[ " $* " == *" $wwn "* ]]; then
+        #echo "ok: skipping $name $wwn, since it was an argument to the script."
+        continue
+    fi
+    root_directory_content=$(grub-fstest /dev/$name ls / 2>/dev/null || true | tr ' ' '\n' | sort | tr '\n' ' ')
+    if [[ $root_directory_content =~ .*boot/.*etc/.* ]]; then
+        echo "FAIL: $name $wwn partitionType=$parttype looks like a Linux root partition on another disk."
+        fail=1
+        continue
+    fi
+    if [[ $root_directory_content =~ .*initrd.*vmlinuz.* ]]; then
+        echo "FAIL: $name $wwn partitionType=$parttype looks like a Linux /boot partition on another disk."
+        fail=1
+        continue
+    fi
+    #echo "ok: $name $wwn $parttype, does not look like root Linux partition."
+done < <(lsblk -r -oNAME,WWN,TYPE,PARTTYPENAME | grep -v NAME | grep -i part)
+if [ $fail -eq 1 ]; then
+    exit 1
+fi
+echo "Looks good. No Linux root partition on other devices"

pkg/services/baremetal/client/ssh/detect-linux-on-another-disk.sh

@@ -20,6 +20,7 @@ package sshclient
 import (
 	"bufio"
 	"bytes"
+	_ "embed"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -35,6 +36,9 @@ const (
 	sshTimeOut time.Duration = 5 * time.Second
 )
 
+//go:embed detect-linux-on-another-disk.sh
+var detectLinuxOnAnotherDiskShellScript string
+
 var downloadFromOciShellScript = `#!/bin/bash
 
 # Copyright 2023 The Kubernetes Authors.
@@ -200,6 +204,7 @@ type Client interface {
 	CleanCloudInitInstances() Output
 	ResetKubeadm() Output
 	UntarTGZ() Output
+	DetectLinuxOnAnotherDisk(sliceOfWwns []string) Output
 }
 
 // Factory is the interface for creating new Client objects.
@@ -482,6 +487,13 @@ func (c *sshClient) ResetKubeadm() Output {
 	return output
 }
 
+func (c *sshClient) DetectLinuxOnAnotherDisk(sliceOfWwns []string) Output {
+	return c.runSSH(fmt.Sprintf(`cat <<'EOF' | bash -s -- %s
+%s
+EOF
+`, strings.Join(sliceOfWwns, " "), detectLinuxOnAnotherDiskShellScript))
+}
+
 func (c *sshClient) UntarTGZ() Output {
 	fileName := "/installimage.tgz"
 	data, err := os.ReadFile(fileName)

pkg/services/baremetal/client/ssh/ssh_client.go

@@ -26,17 +26,17 @@ func Test_removeUselessLinesFromCloudInitOutput(t *testing.T) {
 		want string
 	}{
 		{
-			name: "ignore: 10000K .......... .......... .......... .......... ..........  6%!M(MISSING) 1s",
+			name: "ignore: 10000K ...",
 			s:    "foo\n 10000K .......... .......... .......... .......... ..........  6%!M(MISSING) 1s\nbar",
 			want: "foo\nbar",
 		},
 		{
-			name: "ignore: ^10000K .......... .......... .......... .......... ..........  6%!M(MISSING) 1s",
+			name: "ignore: ^10000K ...2",
 			s:    "foo\n10000K .......... .......... .......... .......... ..........  6%!M(MISSING) 1s\nbar",
 			want: "foo\nbar",
 		},
 		{
-			name: "ignore: Get:17 http://archive.ubuntu.com/ubuntu focal/universe Translation-en [5,124 kB[]",
+			name: "ignore: Get:17 http://...",
 			s:    "foo\nGet:17 http://archive.ubuntu.com/ubuntu focal/universe Translation-en [5,124 kB[]\nbar",
 			want: "foo\nbar",
 		},

pkg/services/baremetal/client/ssh/ssh_client_test.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/syself/hrobot-go/models"
+	"golang.org/x/crypto/ssh"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -514,7 +515,8 @@ func (s *Service) actionRegistering() actionResult {
 
 	// Check hostname with sshClient
 	out := sshClient.GetHostName()
-	if trimLineBreak(out.StdOut) != rescue {
+	hostName := trimLineBreak(out.StdOut)
+	if hostName != rescue {
 		// give the reboot some time until it takes effect
 		if s.hasJustRebooted() {
 			return actionContinue{delay: 2 * time.Second}
@@ -951,14 +953,53 @@ func (s *Service) actionImageInstalling() actionResult {
 
 	s.scope.HetznerBareMetalHost.Spec.Status.SSHStatus.OSKey = &sshKey
 
+	// If there is a Linux OS on an other disk, then the reboot after the provisioning
+	// will likely fail, because the machine boots into the other operating system.
+	// We want detect that early, and not start the provisioning process.
+	out := sshClient.DetectLinuxOnAnotherDisk(s.scope.HetznerBareMetalHost.Spec.RootDeviceHints.ListOfWWN())
+	if out.Err != nil {
+		var exitErr *ssh.ExitError
+		if errors.As(out.Err, &exitErr) && exitErr.ExitStatus() > 0 {
+			// The script detected Linux on an other disk. This is a permanent error.
+			msg := fmt.Sprintf("DetectLinuxOnAnotherDisk failed (permanent error): %s. StdErr: %s (%s)",
+				out.StdOut, out.StdErr, out.Err.Error())
+			conditions.MarkFalse(
+				s.scope.HetznerBareMetalHost,
+				infrav1.ProvisionSucceededCondition,
+				infrav1.LinuxOnOtherDiskFoundReason,
+				clusterv1.ConditionSeverityError,
+				msg,
+			)
+			record.Warn(s.scope.HetznerBareMetalHost, infrav1.LinuxOnOtherDiskFoundReason, msg)
+			s.scope.HetznerBareMetalHost.SetError(infrav1.PermanentError, msg)
+			return actionStop{}
+		}
+
+		// Some other error like connection timeout. Retry again later.
+		// This often during provisioning.
+		msg := fmt.Sprintf("DetectLinuxOnAnotherDisk failed (will retry): %s. StdErr: %s (%s)",
+			out.StdOut, out.StdErr, out.Err.Error())
+		conditions.MarkFalse(
+			s.scope.HetznerBareMetalHost,
+			infrav1.ProvisionSucceededCondition,
+			infrav1.SSHToRescueSystemFailedReason,
+			clusterv1.ConditionSeverityInfo,
+			msg,
+		)
+		record.Event(s.scope.HetznerBareMetalHost, infrav1.SSHToRescueSystemFailedReason, msg)
+		return actionContinue{
+			delay: 10 * time.Second,
+		}
+	}
+
 	autoSetupInput, actionRes := s.createAutoSetupInput(sshClient)
 	if actionRes != nil {
 		return actionRes
 	}
 
 	autoSetup := buildAutoSetup(s.scope.HetznerBareMetalHost.Spec.Status.InstallImage, autoSetupInput)
 
-	out := sshClient.CreateAutoSetup(autoSetup)
+	out = sshClient.CreateAutoSetup(autoSetup)
 	if out.Err != nil {
 		return actionError{err: fmt.Errorf("failed to create autosetup: %q %q %w", out.StdOut, out.StdErr, out.Err)}
 	}