[RSDK-13495] Log orphaned processes (#210)

Author: danielbotros

manager.go

@@ -83,6 +83,42 @@ func NewManager(ctx context.Context, logger logging.Logger, cfg utils.AgentConfi
 		cache:          NewVersionCache(logger),
 		agentStartTime: time.Now(),
 	}
+
+	preexistingProcesses := manager.findPreexistingViamServerProcesses(ctx)
+	if len(preexistingProcesses) > 0 {
+		logger.Warnw(
+			"found process(es) from before agent startup still running; will log every minute while they remain",
+			"processes", preexistingProcesses,
+		)
+		manager.activeBackgroundWorkers.Go(func() {
+			defer utils.Recover(manager.logger, nil)
+
+			remaining := preexistingProcesses
+			ticker := time.NewTicker(time.Minute)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-ticker.C:
+					var stillRunning []utils.Process
+					for _, proc := range remaining {
+						if utils.IsProcessAlive(proc.PID) {
+							stillRunning = append(stillRunning, proc)
+						}
+					}
+					if len(stillRunning) == 0 {
+						manager.logger.Info("all process(es) from before agent startup have exited")
+						return
+					}
+					manager.logger.Warnw("process(es) from before agent startup are still running",
+						"processes", stillRunning)
+					remaining = stillRunning
+				}
+			}
+		})
+	}
+
 	manager.setDebug(cfg.AdvancedSettings.Debug.Get())
 	manager.sysConfig = syscfg.New(
 		ctx,
@@ -803,6 +839,27 @@ func (m *Manager) getVersions() *pb.VersionInfo {
 	return vers
 }
 
+// findPreexistingViamServerProcesses returns any viam-server processes already running when the
+// agent starts, along with their module children. Returns nil if none are found.
+func (m *Manager) findPreexistingViamServerProcesses(ctx context.Context) []utils.Process {
+	pids, err := utils.FindProcessesByName(ctx, viamserver.SubsysName)
+	if err != nil {
+		m.logger.Warnw("error checking for preexisting viam-server processes", "err", err)
+		return nil
+	}
+	var all []utils.Process
+	for _, pid := range pids {
+		all = append(all, utils.Process{PID: pid, Name: viamserver.SubsysName})
+		children, err := utils.FindChildProcesses(ctx, pid)
+		if err != nil {
+			m.logger.Warnw("error checking for module processes under preexisting viam-server", "pid", pid, "err", err)
+			continue
+		}
+		all = append(all, children...)
+	}
+	return all
+}
+
 func (m *Manager) Exit(reason string) {
 	m.logger.Infow(fmt.Sprintf("%s will now exit to be restarted by service manager", SubsystemName), "reason", reason)
 	m.globalCancel()

utils/utils.go

@@ -64,6 +64,12 @@ var (
 	maxBytesForTesting int64
 )
 
+// Process holds identifying info about a running process.
+type Process struct {
+	PID  int
+	Name string
+}
+
 type ViamDirsData struct {
 	Viam  string
 	Bin   string

utils/utils_unix.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"syscall"
 
 	errw "github.com/pkg/errors"
@@ -50,6 +52,60 @@ func KillTree(ctx context.Context, pid int) error {
 	return nil
 }
 
+// FindProcessesByName returns PIDs of all running processes with the given name (exact match).
+func FindProcessesByName(ctx context.Context, name string) ([]int, error) {
+	out, err := exec.CommandContext(ctx, "pgrep", "-x", name).Output()
+	if err != nil {
+		// pgrep exits with code 1 when no processes are found — not an error for us.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var pids []int
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		pid, err := strconv.Atoi(strings.TrimSpace(line))
+		if err != nil {
+			continue
+		}
+		pids = append(pids, pid)
+	}
+	return pids, nil
+}
+
+// FindChildProcesses returns the direct child processes of parentPID.
+func FindChildProcesses(ctx context.Context, parentPID int) ([]Process, error) {
+	//nolint:gosec
+	out, err := exec.CommandContext(ctx, "pgrep", "-l", "-P", strconv.Itoa(parentPID)).Output()
+	if err != nil {
+		// pgrep exits with code 1 when no processes are found — not an error for us.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var children []Process
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		parts := strings.SplitN(strings.TrimSpace(line), " ", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		pid, err := strconv.Atoi(parts[0])
+		if err != nil {
+			continue
+		}
+		children = append(children, Process{PID: pid, Name: parts[1]})
+	}
+	return children, nil
+}
+
+// IsProcessAlive returns true if the process with the given PID is still running.
+func IsProcessAlive(pid int) bool {
+	return syscall.Kill(pid, 0) == nil
+}
+
 func SyncFS(syncPath string) (errRet error) {
 	file, errRet := os.Open(filepath.Dir(syncPath))
 	if errRet != nil {

utils/utils_windows.go

@@ -2,11 +2,14 @@ package utils
 
 import (
 	"context"
+	"fmt"
 	"io/fs"
 	"os"
 	"os/exec"
 	"strconv"
+	"strings"
 	"syscall"
+	"unsafe"
 
 	errw "github.com/pkg/errors"
 	goutils "go.viam.com/utils"
@@ -52,6 +55,90 @@ func SyncFS(syncPath string) error {
 	return nil
 }
 
+// FindProcessesByName returns PIDs of all running processes with the given name (exact match).
+func FindProcessesByName(ctx context.Context, name string) ([]int, error) {
+	//nolint:gosec
+	out, err := exec.CommandContext(ctx, "tasklist", "/FI", "IMAGENAME eq "+name+".exe", "/FO", "CSV", "/NH").Output()
+	if err != nil {
+		return nil, err
+	}
+	var pids []int
+	for _, line := range strings.Split(string(out), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		// CSV format: "name.exe","1234","Console","1","10,000 K"
+		parts := strings.SplitN(line, ",", 3)
+		if len(parts) < 2 {
+			continue
+		}
+		pidStr := strings.Trim(parts[1], `"`)
+		pid, err := strconv.Atoi(pidStr)
+		if err != nil {
+			continue
+		}
+		pids = append(pids, pid)
+	}
+	return pids, nil
+}
+
+// FindChildProcesses returns the direct child processes of parentPID using the Windows API.
+func FindChildProcesses(_ context.Context, parentPID int) ([]Process, error) {
+	snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := windows.CloseHandle(snapshot); err != nil {
+			fmt.Fprintf(os.Stderr, "utils: error closing snapshot handle: %v\n", err)
+		}
+	}()
+
+	var entry windows.ProcessEntry32
+	entry.Size = uint32(unsafe.Sizeof(entry))
+	if err := windows.Process32First(snapshot, &entry); err != nil {
+		return nil, err
+	}
+
+	var children []Process
+	for {
+		if int(entry.ParentProcessID) == parentPID {
+			children = append(children, Process{
+				PID:  int(entry.ProcessID),
+				Name: windows.UTF16ToString(entry.ExeFile[:]),
+			})
+		}
+		if err := windows.Process32Next(snapshot, &entry); err != nil {
+			break
+		}
+	}
+	return children, nil
+}
+
+// stillActive is the value returned by GetExitCodeProcess for a still-running process (STILL_ACTIVE / STATUS_PENDING).
+// See: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess
+const stillActive = 259
+
+// IsProcessAlive returns true if the process with the given PID is still running.
+func IsProcessAlive(pid int) bool {
+	//nolint:gosec
+	h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid))
+	if err != nil {
+		return false
+	}
+	defer func() {
+		if err := windows.CloseHandle(h); err != nil {
+			fmt.Fprintf(os.Stderr, "utils: error closing process handle for pid %d: %v\n", pid, err)
+		}
+	}()
+	var exitCode uint32
+	if err := windows.GetExitCodeProcess(h, &exitCode); err != nil {
+		return false
+	}
+	return exitCode == stillActive
+}
+
 func SignalForTermination(pid int) error {
 	return windows.GenerateConsoleCtrlEvent(syscall.CTRL_BREAK_EVENT, uint32(pid)) //nolint:gosec
 }