Improve systemd packaging smoke tests and stabilize lock contention tests

Author: msander

docs/STATE.md

@@ -81,7 +81,8 @@
   binary starts, boots a transient systemd instance with a drop-in override to
   confirm the packaged unit launches successfully, and confirms assets land at
   the expected paths while gracefully skipping when no container runtime is
-  available.
+  available.  The harness now shares the host cgroup namespace so systemd boots
+  reliably on modern cgroup v2 hosts.
 - A top-level `Makefile` now standardises local builds, cross-compilation for
   `amd64`/`arm64`, and wraps `nfpm` so developers can reproducibly stage
   binaries in `dist/` and generate `.deb`/`.rpm` packages without ad-hoc

packaging/smoke_test.go

@@ -27,6 +27,17 @@ func TestPackagesInstallInContainers(t *testing.T) {
 
 	packages := buildSmokeTestPackages(t)
 
+	cgroupMount := testutil.ContainerMount{Source: "/sys/fs/cgroup", Target: "/sys/fs/cgroup"}
+	if _, err := os.Stat(cgroupMount.Source); err != nil {
+		t.Skipf("skipping container smoke tests: cgroup filesystem not accessible: %v", err)
+	}
+
+	systemdExtraArgs := []string{"--tmpfs", "/run", "--tmpfs", "/run/lock"}
+	switch runtime.Name() {
+	case "docker", "podman":
+		systemdExtraArgs = append([]string{"--cgroupns=host"}, systemdExtraArgs...)
+	}
+
 	type testCase struct {
 		name       string
 		image      string
@@ -36,6 +47,7 @@ func TestPackagesInstallInContainers(t *testing.T) {
 		timeout    time.Duration
 		privileged bool
 		extraArgs  []string
+		mounts     []testutil.ContainerMount
 	}
 
 	cases := []testCase{
@@ -91,7 +103,8 @@ wait "$sd_pid" || true
 `,
 			timeout:    4 * time.Minute,
 			privileged: true,
-			extraArgs:  []string{"--tmpfs", "/run", "--tmpfs", "/run/lock", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"},
+			extraArgs:  append([]string(nil), systemdExtraArgs...),
+			mounts:     []testutil.ContainerMount{cgroupMount},
 		},
 		{
 			name:      "ubuntu-22.04",
@@ -145,7 +158,8 @@ wait "$sd_pid" || true
 `,
 			timeout:    4 * time.Minute,
 			privileged: true,
-			extraArgs:  []string{"--tmpfs", "/run", "--tmpfs", "/run/lock", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"},
+			extraArgs:  append([]string(nil), systemdExtraArgs...),
+			mounts:     []testutil.ContainerMount{cgroupMount},
 		},
 		{
 			name:      "rockylinux-9",
@@ -197,7 +211,8 @@ wait "$sd_pid" || true
 `,
 			timeout:    5 * time.Minute,
 			privileged: true,
-			extraArgs:  []string{"--tmpfs", "/run", "--tmpfs", "/run/lock", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"},
+			extraArgs:  append([]string(nil), systemdExtraArgs...),
+			mounts:     []testutil.ContainerMount{cgroupMount},
 		},
 	}
 
@@ -208,10 +223,11 @@ wait "$sd_pid" || true
 			defer cancel()
 
 			mount := testutil.ContainerMount{Source: packages[tc.format], Target: tc.mountPath, ReadOnly: true}
+			mounts := append([]testutil.ContainerMount{mount}, tc.mounts...)
 			output, err := runtime.Run(ctx, testutil.ContainerRunOptions{
 				Image:      tc.image,
 				Cmd:        []string{"bash", "-lc", tc.script},
-				Mounts:     []testutil.ContainerMount{mount},
+				Mounts:     mounts,
 				Privileged: tc.privileged,
 				ExtraArgs:  tc.extraArgs,
 			})

pkg/orchestrator/runner_test.go

@@ -997,6 +997,7 @@ func TestRunnerEtcdLockPreventsConcurrentReadyNodes(t *testing.T) {
 	lockKey := "/cluster/reboot"
 	nodeNames := []string{"node-a", "node-b"}
 	executor := newTrackingExecutor(len(nodeNames))
+	const lockHoldBeforeAllow = 150 * time.Millisecond
 
 	type nodeHarness struct {
 		name   string
@@ -1102,6 +1103,8 @@ func TestRunnerEtcdLockPreventsConcurrentReadyNodes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("waiting for first reboot command: %v", err)
 	}
+	// Hold the lock briefly so the waiting node attempts acquisition and records contention.
+	time.Sleep(lockHoldBeforeAllow)
 	executor.Allow(first)
 	if completed, err := executor.WaitForCompletion(5 * time.Second); err != nil {
 		t.Fatalf("waiting for %s completion: %v", first, err)
@@ -1116,6 +1119,7 @@ func TestRunnerEtcdLockPreventsConcurrentReadyNodes(t *testing.T) {
 	if second == first {
 		t.Fatalf("expected distinct nodes to execute sequentially, both were %s", second)
 	}
+	time.Sleep(lockHoldBeforeAllow)
 	executor.Allow(second)
 	if completed, err := executor.WaitForCompletion(5 * time.Second); err != nil {
 		t.Fatalf("waiting for %s completion: %v", second, err)
@@ -1181,6 +1185,7 @@ func TestRunnerEtcdLockSerializesThreeNodes(t *testing.T) {
 
 	nodeNames := []string{"node-a", "node-b", "node-c"}
 	executor := newTrackingExecutor(len(nodeNames))
+	const lockHoldBeforeAllow = 150 * time.Millisecond
 
 	type nodeHarness struct {
 		name   string
@@ -1295,6 +1300,12 @@ func TestRunnerEtcdLockSerializesThreeNodes(t *testing.T) {
 		seen[node] = struct{}{}
 		order = append(order, node)
 
+		if len(order) < len(harnesses) {
+			// Hold the lock briefly so peers attempt acquisition and record contention
+			// before the current node completes its simulated reboot.
+			time.Sleep(lockHoldBeforeAllow)
+		}
+
 		executor.Allow(node)
 		if completed, err := executor.WaitForCompletion(5 * time.Second); err != nil {
 			t.Fatalf("waiting for %s completion: %v", node, err)