fix: [sc-109762] Isolate the Linux auto-update helper from its own systemd cgroup

The --update helper inherited the running service's cgroup, so calling
systemctl stop on its own unit let KillMode=control-group kill the helper
along with the service before it could restart it, leaving the endpoint
offline with no auto-recovery. The helper now runs in its own transient
systemd scope (systemd-run --scope) so it survives the stop.

Adds integration-test coverage that asserts the helper actually logs a
restart, the service is active, and it resubscribes after a Linux
auto-update, closing a gap where the existing check could pass even when
the helper never restarted anything.

Author: mlataza

.github/workflows/integration-test.yml

@@ -290,6 +290,25 @@ jobs:
           binary: ${{ matrix.it_binary }}
           args: --update --org-id ${{ vars.IT_ORG_ID }} --github-token ${{ secrets.GITHUB_TOKEN }}
 
+      # Captured before the real auto-update fires, so the assertions below
+      # measure what that update did rather than the restart the IT binary
+      # install above already caused.
+      - name: Capture pre-auto-update baseline counts
+        id: autoupdate_baseline
+        shell: pwsh
+        env:
+          LOG_FILE: ${{ matrix.log_file }}
+        run: |
+          $content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
+          function Get-PatternCount($text, $needle) {
+            if ($text) { ([regex]::Matches($text, [regex]::Escape($needle))).Count } else { 0 }
+          }
+          $started    = Get-PatternCount $content "Service started"
+          $subscribed = Get-PatternCount $content "Subscribed to messages"
+          "started=$started"       >> $env:GITHUB_OUTPUT
+          "subscribed=$subscribed" >> $env:GITHUB_OUTPUT
+          Write-Output "Baseline counts -> started=$started subscribed=$subscribed"
+
       - name: Wait for auto updater to trigger and update
         uses: ./.github/actions/wait-for-log-line
         with:
@@ -301,6 +320,82 @@ jobs:
         with:
           log_file: ${{ matrix.log_file }}
 
+      # ---- Linux systemd cgroup-kill regression (sc-109762) ----
+      # verify-auto-update above only checks the log for "Updating agent" and
+      # that the last "Agent Smith started" line isn't the IT version - but
+      # that line is logged by the --update helper on its own startup, before
+      # it ever stops the old service, so it already names the new version
+      # even when the helper is killed moments later and never restarts
+      # anything. On Linux, the helper used to inherit the running service's
+      # systemd cgroup, so calling `systemctl stop` on its own unit triggered
+      # KillMode=control-group and killed the helper along with the service it
+      # asked to stop - the endpoint was left offline with the old binary
+      # still in place and no automatic recovery. The real proof the update
+      # finished is the helper's own "Service started" line, logged only after
+      # it replaced the binary and called svc.Start() - the exact point the
+      # cgroup kill used to cut it off. Counts are compared as deltas because
+      # "Service started" also appears once from installing the IT binary
+      # above, and "Subscribed to messages" already has lines from earlier
+      # scenarios.
+      - name: Assert the update helper survived to restart the service
+        shell: pwsh
+        env:
+          LOG_FILE: ${{ matrix.log_file }}
+          BASELINE: ${{ steps.autoupdate_baseline.outputs.started }}
+        run: |
+          $baseline = [int]$env:BASELINE
+          for ($i = 1; $i -le 30; $i++) {
+            $content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
+            $count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Service started"))).Count } else { 0 }
+            if ($count -gt $baseline) {
+              Write-Output "Update helper restarted the service (count $count > baseline $baseline) after ~$($i * 2)s"
+              exit 0
+            }
+            Start-Sleep -Seconds 2
+          }
+          Write-Output "---- last 60 lines of the agent log ----"
+          Get-Content $env:LOG_FILE -Tail 60 -ErrorAction SilentlyContinue
+          Write-Error "Update helper never logged 'Service started' after the auto-update; it was likely killed mid-update (count stayed at $baseline)"
+          exit 1
+
+      - name: Check service is active after auto-update
+        uses: ./.github/actions/check-service
+        with:
+          service: ${{ matrix.service }}
+
+      - name: Wait for fresh subscription after auto-update
+        shell: pwsh
+        env:
+          LOG_FILE: ${{ matrix.log_file }}
+          BASELINE: ${{ steps.autoupdate_baseline.outputs.subscribed }}
+        run: |
+          $baseline = [int]$env:BASELINE
+          for ($i = 1; $i -le 30; $i++) {
+            $content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
+            $count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
+            if ($count -gt $baseline) {
+              Write-Output "Agent resubscribed after auto-update (count $count > baseline $baseline) after ~$($i * 2)s"
+              exit 0
+            }
+            Start-Sleep -Seconds 2
+          }
+          Write-Error "Agent never resubscribed after auto-update (subscribed count stayed at $baseline)"
+          exit 1
+
+      - name: Send command after auto-update
+        uses: ./.github/actions/send-command
+        with:
+          trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
+          device_id: ${{ steps.config.outputs.device_id }}
+          commands: ${{ matrix.success_commands }}
+
+      - name: Check logs for command completion after auto-update
+        uses: ./.github/actions/assert-log-contains
+        with:
+          log_file: ${{ matrix.log_file }}
+          patterns: ${{ matrix.success_patterns }}
+          wait_seconds: "15"
+
       - name: Update logging level to debug
         uses: ./.github/actions/run-agent
         with:

CLAUDE.md

@@ -61,7 +61,15 @@ Required tools:
   leave an endpoint offline. The agent executable and config file are written to a
   temp file and atomically renamed into place, so a failed write leaves the
   previous file byte-identical. See the README's "Waiting for the Old Agent
-  Process to Exit" section.
+  Process to Exit" section. On Linux, the `--update` helper that performs this
+  flow is launched in its own transient systemd scope
+  (`cmd/agent_smith/run_command_linux.go`, `systemd-run --scope --collect`)
+  rather than as a plain child of the running service, so it is never a member
+  of that service's cgroup — otherwise `systemctl stop`'s default
+  `KillMode=control-group` kills the helper along with the service it just
+  stopped, before it can restart it, leaving the endpoint offline with no
+  automatic recovery. See the README's "Surviving Its Own systemd Stop
+  (Linux)" section.
 
 - **internal/agent/**: Device configuration, installation paths, and OS-specific host information. Auto-update installers are downloaded into `<data directory>/updates` (a `0700` directory the agent owns) rather than the shared system temp directory, and `SweepStaleInstallers` reclaims installer binaries older than 24 hours at service startup — from that directory and from the legacy temp location — so the binaries a detached installer necessarily leaves behind stop accumulating one per update. See the README's "Reclaiming Downloaded Installer Binaries" section. The auto-update retry schedule is capped (1 hour, or a quarter of the check interval when shorter) and jittered (±25%) via `utils.JitteredBackoff`, the same helper the postback retry schedule uses, so the doubling cannot overflow into a negative sleep that busy-spins and a fleet-wide release-endpoint outage cannot produce a synchronized retry storm. See the README's "Capped and Jittered Auto-Update Retries" section.
 - **internal/interpreter/**: Command execution engine supporting both PowerShell and Bash interpreters  

README.md

@@ -536,6 +536,34 @@ pattern the postback spool uses. An interrupted or failed write therefore leaves
 the previous file byte-identical rather than truncated: the endpoint keeps running
 the old agent instead of a binary that cannot start.
 
+### Surviving Its Own systemd Stop (Linux)
+
+The `--update` helper that an auto-update spawns has to stop the running
+service before it can replace the binary and start it again. On Linux that
+service runs as a systemd unit, and systemd's default `KillMode=control-group`
+tears down every process in the unit's cgroup — not just its main process —
+when the unit is stopped. The helper is launched as a child of the running
+service, so without intervention it inherits that cgroup: the moment it calls
+`systemctl stop` on its own unit, systemd kills the helper along with the
+service it just asked to stop, mid-update. The service is left stopped, the
+binary and config were never touched, and — because the kill is a signal, not
+a normal return — the helper's own deferred recovery never runs either.
+`Restart=always` does not help: the unit was stopped by an explicit
+`systemctl stop`, which systemd treats as a clean, intentional exit, not the
+unexpected one `Restart=` reacts to.
+
+The helper now runs inside its own transient systemd **scope**
+(`systemd-run --scope --collect`) rather than as a plain child process, so it
+is never a member of the unit's cgroup in the first place. Stopping the unit
+it was launched from tears down only that unit's cgroup; the helper's scope is
+untouched, so it survives to replace the binary, update the config, and start
+the service again — the same flow already used on Windows and macOS. macOS
+needed no equivalent change: launchd tears down a stopped job by BSD process
+group (`killpg`), and the `Setsid` the helper already sets moves it into a new
+process group, which is enough to escape that teardown. Linux's cgroup-based
+`KillMode` is inherited across `fork()` and untouched by `setsid()`, so the
+same call that protects the helper on macOS does not protect it on Linux.
+
 ### Capped and Jittered Auto-Update Retries
 
 When an update check or download fails, the agent retries on an exponential

cmd/agent_smith/run_command_darwin.go

@@ -0,0 +1,24 @@
+//go:build darwin
+
+package main
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+// detachedCommand launches the auto-update helper detached from the running
+// service process. Setsid alone is sufficient on macOS: launchd tears down a
+// stopped job by BSD process group (killpg), and setsid() moves the helper
+// into a brand new session/process group, so it is already outside the group
+// launchd kills. Linux's KillMode=control-group instead kills by cgroup
+// membership, which setsid() does not change, so Linux needs a different
+// mechanism — see run_command_linux.go.
+func detachedCommand(path string, args []string, stdout, stderr *os.File) *exec.Cmd {
+	cmd := exec.Command(path, args...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
+	return cmd
+}

cmd/agent_smith/run_command_darwin_test.go

@@ -0,0 +1,34 @@
+//go:build darwin
+
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+func TestDetachedCommandRunsDirectlyOnDarwin(t *testing.T) {
+	path := "/opt/agent/rewst_agent_config.mac-os.bin"
+	args := []string{"--update", "--org-id", "abc"}
+	cmd := detachedCommand(path, args, os.Stdout, os.Stderr)
+
+	if cmd.Path != path {
+		t.Fatalf("Path = %q, want %q", cmd.Path, path)
+	}
+	wantArgs := append([]string{path}, args...)
+	if len(cmd.Args) != len(wantArgs) {
+		t.Fatalf("Args = %v, want %v", cmd.Args, wantArgs)
+	}
+	for i, want := range wantArgs {
+		if cmd.Args[i] != want {
+			t.Fatalf("Args[%d] = %q, want %q", i, cmd.Args[i], want)
+		}
+	}
+
+	if cmd.SysProcAttr == nil {
+		t.Fatalf("SysProcAttr = nil, want non-nil")
+	}
+	if !cmd.SysProcAttr.Setsid {
+		t.Fatalf("Setsid = false, want true")
+	}
+}

cmd/agent_smith/run_command_linux.go

@@ -0,0 +1,29 @@
+//go:build linux
+
+package main
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+// detachedCommand launches the auto-update helper (path, args) inside its own
+// transient systemd scope instead of running it as a direct child of the
+// service process. The helper's job is to call `systemctl stop` on the very
+// unit it was launched from; a plain child inherits that unit's cgroup, and
+// systemd's default KillMode=control-group kills every process in the
+// cgroup — including the helper itself — the moment the stop is issued, before
+// it can replace the binary or start the service again. `systemd-run --scope`
+// creates a new transient scope (and cgroup) for the helper up front, so it is
+// never a member of the unit's cgroup and survives stopping it. `--collect`
+// releases the transient scope unit's bookkeeping once the helper exits so
+// scopes don't accumulate one per update.
+func detachedCommand(path string, args []string, stdout, stderr *os.File) *exec.Cmd {
+	scopeArgs := append([]string{"--scope", "--collect", "--quiet", "--", path}, args...)
+	cmd := exec.Command("systemd-run", scopeArgs...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
+	return cmd
+}

cmd/agent_smith/run_command_linux_test.go

@@ -0,0 +1,49 @@
+//go:build linux
+
+package main
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestDetachedCommandWrapsInSystemdScope(t *testing.T) {
+	cmd := detachedCommand("/opt/agent/rewst_agent_config.linux.bin", []string{"--update", "--org-id", "abc"}, os.Stdout, os.Stderr)
+
+	wantPath := "systemd-run"
+	if got := cmd.Path; got != wantPath && !strings.HasSuffix(got, "/"+wantPath) {
+		t.Fatalf("Path = %q, want %q (or a resolved path ending in it)", got, wantPath)
+	}
+
+	wantArgs := []string{
+		"systemd-run",
+		"--scope",
+		"--collect",
+		"--quiet",
+		"--",
+		"/opt/agent/rewst_agent_config.linux.bin",
+		"--update",
+		"--org-id",
+		"abc",
+	}
+	if len(cmd.Args) != len(wantArgs) {
+		t.Fatalf("Args = %v, want %v", cmd.Args, wantArgs)
+	}
+	for i, want := range wantArgs {
+		if cmd.Args[i] != want {
+			t.Fatalf("Args[%d] = %q, want %q (full: %v)", i, cmd.Args[i], want, cmd.Args)
+		}
+	}
+
+	if cmd.SysProcAttr == nil {
+		t.Fatalf("SysProcAttr = nil, want non-nil")
+	}
+	if !cmd.SysProcAttr.Setsid {
+		t.Fatalf("Setsid = false, want true")
+	}
+}
+
+func endsWith(s, suffix string) bool {
+	return len(s) >= len(suffix) && s[len(s)-len(suffix):] == suffix
+}