Resolve selinux installation issues (#2394)

hedge-sparrow · web-flow · commit 80fb2ff2f759 · 2025-07-31T09:23:45.000-07:00
* run restorecon after materialising files * don't care about selinux enforcing * set specific bin_t context on bins dir * path * Correct match group * add selinux preflight * message formatting * explicitly check for restorecon binary * Add selinux test This builds on the existing airgapped testing. due to some limitations in our alma images I had to change all script calls to absolute paths. * bump troubleshoot tag * remove path env from sudo prefix * move selinux setup to hostutils interface * mock * Revert "remove path env from sudo prefix" This reverts commit 9b932da.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -910,6 +910,7 @@ jobs:
         test:
           - TestResetAndReinstallAirgap
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
           - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade
diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml
@@ -569,6 +569,7 @@ jobs:
         test:
           - TestResetAndReinstallAirgap
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
           - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade
diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go
@@ -205,7 +205,15 @@ func copyScriptsToNode(node Node) error {
 }
 
 func getSSHEndpoint(nodeID string) (string, error) {
-	output, err := exec.Command("replicated", "vm", "ssh-endpoint", nodeID).CombinedOutput()
+	args := []string{
+		"vm",
+		"ssh-endpoint",
+		nodeID,
+	}
+	if user := os.Getenv("CMX_SSH_USERNAME"); user != "" {
+		args = append(args, "--username", user)
+	}
+	output, err := exec.Command("replicated", args...).CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("%v: %s", err, string(output))
 	}
@@ -335,7 +343,7 @@ func runCommandOnNode(node Node, line []string, envs ...map[string]string) (stri
 			line = append([]string{fmt.Sprintf("%s=%s", k, v)}, line...)
 		}
 	}
-	line = append([]string{"sudo"}, line...)
+	line = append([]string{"sudo", "PATH=$PATH:/usr/local/bin"}, line...)
 
 	cmd := exec.Command("ssh", append(sshArgs(), node.sshEndpoint, strings.Join(line, " "))...)
 	var stdout, stderr bytes.Buffer
@@ -363,7 +371,7 @@ func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (st
 
 func (c *Cluster) SetupPlaywright(envs ...map[string]string) error {
 	c.t.Logf("%s: bypassing kurl-proxy", time.Now().Format(time.RFC3339))
-	_, stderr, err := c.RunCommandOnNode(0, []string{"bypass-kurl-proxy.sh"}, envs...)
+	_, stderr, err := c.RunCommandOnNode(0, []string{"/usr/local/bin/bypass-kurl-proxy.sh"}, envs...)
 	if err != nil {
 		return fmt.Errorf("bypass kurl-proxy: %v: %s", err, string(stderr))
 	}
@@ -401,7 +409,7 @@ func (c *Cluster) generateSupportBundle(envs ...map[string]string) {
 		go func(i int, wg *sync.WaitGroup) {
 			defer wg.Done()
 			c.t.Logf("%s: generating host support bundle from node %d", time.Now().Format(time.RFC3339), i)
-			if stdout, stderr, err := c.RunCommandOnNode(i, []string{"collect-support-bundle-host.sh"}, envs...); err != nil {
+			if stdout, stderr, err := c.RunCommandOnNode(i, []string{"/usr/local/bin/collect-support-bundle-host.sh"}, envs...); err != nil {
 				c.t.Logf("stdout: %s", stdout)
 				c.t.Logf("stderr: %s", stderr)
 				c.t.Logf("fail to generate support bundle from node %d: %v", i, err)
@@ -419,7 +427,7 @@ func (c *Cluster) generateSupportBundle(envs ...map[string]string) {
 	}
 
 	c.t.Logf("%s: generating cluster support bundle from node %d", time.Now().Format(time.RFC3339), c.supportBundleNodeIndex)
-	if stdout, stderr, err := c.RunCommandOnNode(c.supportBundleNodeIndex, []string{"collect-support-bundle-cluster.sh"}, envs...); err != nil {
+	if stdout, stderr, err := c.RunCommandOnNode(c.supportBundleNodeIndex, []string{"/usr/local/bin/collect-support-bundle-cluster.sh"}, envs...); err != nil {
 		c.t.Logf("stdout: %s", stdout)
 		c.t.Logf("stderr: %s", stderr)
 		c.t.Logf("fail to generate cluster support bundle from node %d: %v", c.supportBundleNodeIndex, err)
diff --git a/e2e/install_test.go b/e2e/install_test.go
@@ -694,6 +694,95 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestSingleNodeAirgapUpgradeSelinux(t *testing.T) {
+	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
+	tc := cmx.NewCluster(&cmx.ClusterInput{
+		T:            t,
+		Nodes:        1,
+		Distribution: "almalinux",
+		Version:      "8",
+	})
+	defer tc.Cleanup()
+
+	t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339))
+	initialVersion := fmt.Sprintf("appver-%s-previous-k0s", os.Getenv("SHORT_SHA"))
+	runInParallel(t,
+		func(t *testing.T) error {
+			return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID)
+		}, func(t *testing.T) error {
+			return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID)
+		},
+	)
+
+	t.Logf("%s: airgapping cluster", time.Now().Format(time.RFC3339))
+	if err := tc.Airgap(); err != nil {
+		t.Fatalf("failed to airgap cluster: %v", err)
+	}
+
+	t.Logf("%s: creating /.autorelabel file for SELinux relabeling", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"touch", "/.autorelabel"}); err != nil {
+		t.Fatalf("fail to create /.autorelabel file on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: rebooting VM for SELinux relabeling", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"reboot"}); err != nil {
+		t.Fatalf("fail to reboot node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: waiting for node to reboot", time.Now().Format(time.RFC3339))
+	tc.WaitForReboot()
+
+	t.Logf("%s: setting selinux to Enforcing mode", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"setenforce 1"}); err != nil {
+		t.Fatalf("fail to set selinux to Enforcing mode %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339))
+	line := []string{"/usr/local/bin/airgap-prepare.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to prepare airgap files on node %s: %v: %s: %s", tc.Nodes[0], err, stdout, stderr)
+	}
+
+	installSingleNodeWithOptions(t, tc, installOptions{
+		isAirgap:                true,
+		version:                 initialVersion,
+		localArtifactMirrorPort: "50001", // choose an alternate lam port
+	})
+
+	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
+	line = []string{"/usr/local/bin/check-airgap-installation-state.sh", initialVersion, k8sVersionPrevious()}
+	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	checkNodeJoinCommand(t, tc, 0)
+
+	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
+	line = []string{"/usr/local/bin/airgap-update.sh"}
+	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to run airgap update: %v", err)
+	}
+
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
+	}
+
+	checkPostUpgradeState(t, tc)
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 	t.Parallel()
 
diff --git a/e2e/shared.go b/e2e/shared.go
@@ -76,9 +76,9 @@ func installSingleNodeWithOptions(t *testing.T, tc cluster.Cluster, opts install
 	line := []string{}
 
 	if opts.isAirgap {
-		line = append(line, "single-node-airgap-install.sh")
+		line = append(line, "/usr/local/bin/single-node-airgap-install.sh")
 	} else {
-		line = append(line, "single-node-install.sh")
+		line = append(line, "/usr/local/bin/single-node-install.sh")
 		// the cli/ui option is currently only applicable for online installs
 		if opts.viaCLI {
 			line = append(line, "cli")
@@ -136,7 +136,7 @@ func checkInstallationState(t *testing.T, tc cluster.Cluster) {
 }
 
 func checkInstallationStateWithOptions(t *testing.T, tc cluster.Cluster, opts installationStateOptions) {
-	line := []string{"check-installation-state.sh"}
+	line := []string{"/usr/local/bin/check-installation-state.sh"}
 	if opts.version != "" {
 		line = append(line, opts.version)
 	} else {
@@ -233,23 +233,23 @@ func joinWorkerNodeWithOptions(t *testing.T, tc cluster.Cluster, node int, opts
 
 func waitForNodes(t *testing.T, tc cluster.Cluster, nodes int, envs map[string]string, args ...string) {
 	t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
-	stdout, stderr, err := tc.RunCommandOnNode(0, append([]string{"wait-for-ready-nodes.sh", fmt.Sprintf("%d", nodes)}, args...), envs)
+	stdout, stderr, err := tc.RunCommandOnNode(0, append([]string{"/usr/local/bin/wait-for-ready-nodes.sh", fmt.Sprintf("%d", nodes)}, args...), envs)
 	if err != nil {
 		t.Fatalf("fail to wait for ready nodes: %v: %s: %s", err, stdout, stderr)
 	}
 }
 
 func checkWorkerProfile(t *testing.T, tc cluster.Cluster, node int) {
 	t.Logf("checking worker profile on node %d", node)
-	line := []string{"check-worker-profile.sh"}
+	line := []string{"/usr/local/bin/check-worker-profile.sh"}
 	if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
 		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", node, err, stdout, stderr)
 	}
 }
 
 func checkNodeJoinCommand(t *testing.T, tc cluster.Cluster, node int) {
 	t.Logf("node join command generation on node %d", node)
-	line := []string{"check-node-join-command.sh"}
+	line := []string{"/usr/local/bin/check-node-join-command.sh"}
 	if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
 		t.Fatalf("fail to check if node join command is generated successfully on node %d: %v: %s: %s", node, err, stdout, stderr)
 	}
@@ -261,7 +261,7 @@ func downloadECRelease(t *testing.T, tc cluster.Cluster, node int) {
 
 func downloadECReleaseWithOptions(t *testing.T, tc cluster.Cluster, node int, opts downloadECReleaseOptions) {
 	t.Logf("%s: downloading embedded cluster release on node %d", time.Now().Format(time.RFC3339), node)
-	line := []string{"vandoor-prepare.sh"}
+	line := []string{"/usr/local/bin/vandoor-prepare.sh"}
 
 	if opts.version != "" {
 		line = append(line, opts.version)
@@ -292,7 +292,7 @@ func resetInstallationWithOptions(t *testing.T, tc cluster.Cluster, node int, op
 
 func resetInstallationWithError(t *testing.T, tc cluster.Cluster, node int, opts resetInstallationOptions) (string, string, error) {
 	t.Logf("%s: resetting the installation on node %d", time.Now().Format(time.RFC3339), node)
-	line := []string{"reset-installation.sh"}
+	line := []string{"/usr/local/bin/reset-installation.sh"}
 	if opts.force {
 		line = append(line, "--force")
 	}
@@ -304,7 +304,7 @@ func checkPostUpgradeState(t *testing.T, tc cluster.Cluster) {
 }
 
 func checkPostUpgradeStateWithOptions(t *testing.T, tc cluster.Cluster, opts postUpgradeStateOptions) {
-	line := []string{"check-postupgrade-state.sh"}
+	line := []string{"/usr/local/bin/check-postupgrade-state.sh"}
 
 	if opts.k8sVersion != "" {
 		line = append(line, opts.k8sVersion)
diff --git a/pkg-new/hostutils/initialize.go b/pkg-new/hostutils/initialize.go
@@ -39,6 +39,16 @@ func (h *HostUtils) ConfigureHost(ctx context.Context, rc runtimeconfig.RuntimeC
 		}
 	}
 
+	h.logger.Debugf("configuring selinux fcontext")
+	if err := h.ConfigureSELinuxFcontext(rc); err != nil {
+		h.logger.Debugf("unable to configure selinux fcontext: %v", err)
+	}
+
+	h.logger.Debugf("restoring selinux context")
+	if err := h.RestoreSELinuxContext(rc); err != nil {
+		h.logger.Debugf("unable to restore selinux context: %v", err)
+	}
+
 	h.logger.Debugf("configuring sysctl")
 	if err := h.ConfigureSysctl(); err != nil {
 		h.logger.Debugf("unable to configure sysctl: %v", err)
diff --git a/pkg-new/hostutils/interface.go b/pkg-new/hostutils/interface.go
@@ -28,6 +28,8 @@ type HostUtilsInterface interface {
 	CreateSystemdUnitFiles(ctx context.Context, logger logrus.FieldLogger, rc runtimeconfig.RuntimeConfig, isWorker bool) error
 	WriteLocalArtifactMirrorDropInFile(rc runtimeconfig.RuntimeConfig) error
 	AddInsecureRegistry(registry string) error
+	ConfigureSELinuxFcontext(rc runtimeconfig.RuntimeConfig) error
+	RestoreSELinuxContext(rc runtimeconfig.RuntimeConfig) error
 }
 
 // Convenience functions
@@ -72,3 +74,11 @@ func WriteLocalArtifactMirrorDropInFile(rc runtimeconfig.RuntimeConfig) error {
 func AddInsecureRegistry(registry string) error {
 	return h.AddInsecureRegistry(registry)
 }
+
+func ConfigureSELinuxFcontext(rc runtimeconfig.RuntimeConfig) error {
+	return h.ConfigureSELinuxFcontext(rc)
+}
+
+func RestoreSELinuxContext(rc runtimeconfig.RuntimeConfig) error {
+	return h.RestoreSELinuxContext(rc)
+}
diff --git a/pkg-new/hostutils/mock.go b/pkg-new/hostutils/mock.go
@@ -15,6 +15,18 @@ type MockHostUtils struct {
 	mock.Mock
 }
 
+// ConfigureSELinuxFcontext implements HostUtilsInterface.
+func (m *MockHostUtils) ConfigureSELinuxFcontext(rc runtimeconfig.RuntimeConfig) error {
+	args := m.Called(rc)
+	return args.Error(0)
+}
+
+// RestoreSELinuxContext implements HostUtilsInterface.
+func (m *MockHostUtils) RestoreSELinuxContext(rc runtimeconfig.RuntimeConfig) error {
+	args := m.Called(rc)
+	return args.Error(0)
+}
+
 // ConfigureHost mocks the ConfigureHost method
 func (m *MockHostUtils) ConfigureHost(ctx context.Context, rc runtimeconfig.RuntimeConfig, opts InitForInstallOptions) error {
 	args := m.Called(ctx, rc, opts)
diff --git a/pkg-new/hostutils/selinux.go b/pkg-new/hostutils/selinux.go
@@ -0,0 +1,50 @@
+package hostutils
+
+import (
+	"os/exec"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
+)
+
+func (h *HostUtils) ConfigureSELinuxFcontext(rc runtimeconfig.RuntimeConfig) error {
+	h.logger.Debugln("checking for semanage binary in $PATH")
+	if _, err := exec.LookPath("semanage"); err != nil {
+		h.logger.Debugln("semanage not found in $PATH")
+		return nil
+	}
+
+	h.logger.Debugf("setting selinux fcontext for embedded-cluster binary directory to bin_t")
+	args := []string{
+		"fcontext",
+		"-a",
+		"-s",
+		"system_u",
+		"-t",
+		"bin_t",
+		rc.EmbeddedClusterBinsSubDir() + "(/.*)?",
+	}
+	out, err := exec.Command("semanage", args...).CombinedOutput()
+	if err != nil {
+		h.logger.Debugf("unable to set contexts on binary directory: %v", err)
+		h.logger.Debugln(string(out))
+	}
+
+	return nil
+}
+
+func (h *HostUtils) RestoreSELinuxContext(rc runtimeconfig.RuntimeConfig) error {
+	h.logger.Debugln("checking for restorecon binary in $PATH")
+	if _, err := exec.LookPath("restorecon"); err != nil {
+		h.logger.Debugln("restorecon not found in $PATH")
+		return nil
+	}
+
+	h.logger.Debugf("relabeling embedded-cluster data directory with restorecon")
+	out, err := exec.Command("restorecon", "-RvF", rc.EmbeddedClusterHomeDirectory()).CombinedOutput()
+	if err != nil {
+		h.logger.Debugf("unable to run restorecon: %v", err)
+		h.logger.Debugln(string(out))
+	}
+
+	return nil
+}
diff --git a/pkg-new/preflights/host-preflight.yaml b/pkg-new/preflights/host-preflight.yaml
