feat(restarter): add k8s distro-specific restarters

Signed-off-by: Vaughn Dice <[email protected]>

Author: vdice

internal/containerd/restart_unix.go

@@ -22,35 +22,146 @@ package containerd
 import (
 	"fmt"
 	"log/slog"
+	"os"
+	"os/exec"
+	"regexp"
 	"syscall"
 
 	"github.com/mitchellh/go-ps"
 )
 
 var psProcesses = ps.Processes
 
-type restarter struct{}
+type defaultRestarter struct{}
 
-func NewRestarter() Restarter {
-	return restarter{}
+func NewDefaultRestarter() Restarter {
+	return defaultRestarter{}
 }
 
-func (c restarter) Restart() error {
-	pid, err := getPid()
+func (c defaultRestarter) Restart() error {
+	// If systemctl exists, use that, otherwise go pid
+	if UsesSystemd() {
+		out, err := nsenterCmd("systemctl", "restart", "containerd").CombinedOutput()
+		slog.Debug(string(out))
+		if err != nil {
+			return fmt.Errorf("unable to restart containerd: %w", err)
+		}
+	} else {
+		pid, err := getPid("containerd")
+		if err != nil {
+			return err
+		}
+		slog.Debug("found containerd process", "pid", pid)
+
+		err = syscall.Kill(pid, syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("failed to send SIGHUP to containerd: %w", err)
+		}
+	}
+
+	return nil
+}
+
+type K0sRestarter struct{}
+
+func (c K0sRestarter) Restart() error {
+	// First, collect systemd units to determine which mode k0s is running in, eg
+	// k0sworker or k0scontroller
+	units, err := nsenterCmd("systemctl", "list-units").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("unable to list systemd units: %w", err)
+	}
+	service := regexp.MustCompile("k0sworker|k0scontroller").FindString(string(units))
+
+	out, err := nsenterCmd("systemctl", "restart", service).CombinedOutput()
+	slog.Debug(string(out))
+	if err != nil {
+		return fmt.Errorf("unable to restart %s: %w", service, err)
+	}
+
+	return nil
+}
+
+type K3sRestarter struct{}
+
+func (c K3sRestarter) Restart() error {
+	// This restarter will be used both for stock K3s distros
+	// using systemd as well as K3d, which does not.
+	if UsesSystemd() {
+		out, err := nsenterCmd("systemctl", "restart", "k3s").CombinedOutput()
+		slog.Debug(string(out))
+		if err != nil {
+			return fmt.Errorf("unable to restart k3s: %w", err)
+		}
+	} else {
+		// TODO: this approach still leads to the behavior mentioned in https://github.com/spinframework/runtime-class-manager/issues/140:
+		// The first pod's provisioner container exits with code 255, leading to pod status Unknown,
+		// followed by the subsequent pod's provisioner container no-op-ing and finishing with status Completed.
+		pid, err := getPid("k3s")
+		if err != nil {
+			return err
+		}
+		slog.Debug("found k3s process", "pid", pid)
+
+		err = syscall.Kill(pid, syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("failed to send SIGHUP to k3s: %w", err)
+		}
+	}
+
+	return nil
+}
+
+type MicroK8sRestarter struct{}
+
+func (c MicroK8sRestarter) Restart() error {
+	out, err := nsenterCmd("systemctl", "restart", "snap.microk8s.daemon-containerd").CombinedOutput()
+	slog.Debug(string(out))
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to restart snap.microk8s.daemon-containerd: %w", err)
 	}
-	slog.Debug("found containerd process", "pid", pid)
 
-	err = syscall.Kill(pid, syscall.SIGHUP)
+	return nil
+}
 
+type RKE2Restarter struct{}
+
+func (c RKE2Restarter) Restart() error {
+	// First, collect systemd units to determine which mode rke2 is running in, eg
+	// rke2-agent or rke2-server
+	units, err := nsenterCmd("systemctl", "list-units").CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("failed to send SIGHUP to containerd: %w", err)
+		return fmt.Errorf("unable to list systemd units: %w", err)
 	}
+	service := regexp.MustCompile("rke2-agent|rke2-server").FindString(string(units))
+
+	out, err := nsenterCmd("systemctl", "restart", service).CombinedOutput()
+	slog.Debug(string(out))
+	if err != nil {
+		return fmt.Errorf("unable to restart %s: %w", service, err)
+	}
+
 	return nil
 }
 
-func getPid() (int, error) {
+// TODO: lifted and amended from https://github.com/spinframework/runtime-class-manager/pull/387
+//
+// UsesSystemd checks if the system is using systemd
+func UsesSystemd() bool {
+	cmd := nsenterCmd("systemctl", "list-units", "|", "grep", "-q", "containerd.service")
+	if err := cmd.Run(); err != nil {
+		slog.Debug("Error with systemctl: \n", "error", err)
+		return false
+	}
+	return true
+}
+
+func nsenterCmd(cmd ...string) *exec.Cmd {
+	return exec.Command("nsenter",
+		append([]string{fmt.Sprintf("-m/%s/proc/1/ns/mnt", os.Getenv("HOST_ROOT")), "--"}, cmd...)...) // #nosec G204
+}
+
+func getPid(executable string) (int, error) {
 	processes, err := psProcesses()
 	if err != nil {
 		return 0, fmt.Errorf("could not get processes: %w", err)
@@ -59,13 +170,13 @@ func getPid() (int, error) {
 	var containerdProcesses = []ps.Process{}
 
 	for _, process := range processes {
-		if process.Executable() == "containerd" {
+		if process.Executable() == executable {
 			containerdProcesses = append(containerdProcesses, process)
 		}
 	}
 
 	if len(containerdProcesses) != 1 {
-		return 0, fmt.Errorf("need exactly one containerd process, found: %d", len(containerdProcesses))
+		return 0, fmt.Errorf("need exactly one %s process, found: %d", executable, len(containerdProcesses))
 	}
 
 	return containerdProcesses[0].Pid(), nil

internal/preset/preset.go

@@ -24,7 +24,7 @@ type Env struct {
 var Default = Settings{
 	ConfigPath: "/etc/containerd/config.toml",
 	Setup:      func(_ Env) error { return nil },
-	Restarter:  containerd.NewRestarter(),
+	Restarter:  containerd.NewDefaultRestarter(),
 }
 
 func (s Settings) WithConfigPath(path string) Settings {
@@ -37,9 +37,16 @@ func (s Settings) WithSetup(setup func(env Env) error) Settings {
 	return s
 }
 
-var MicroK8s = Default.WithConfigPath("/var/snap/microk8s/current/args/containerd-template.toml")
+func (s Settings) WithRestarter(restarter containerd.Restarter) Settings {
+	s.Restarter = restarter
+	return s
+}
+
+var MicroK8s = Default.WithConfigPath("/var/snap/microk8s/current/args/containerd-template.toml").
+	WithRestarter(containerd.MicroK8sRestarter{})
 
 var RKE2 = Default.WithConfigPath("/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl").
+	WithRestarter(containerd.RKE2Restarter{}).
 	WithSetup(func(env Env) error {
 		_, err := env.HostFs.Stat(env.ConfigPath)
 		if err == nil {
@@ -75,9 +82,11 @@ var RKE2 = Default.WithConfigPath("/var/lib/rancher/rke2/agent/etc/containerd/co
 		return err
 	})
 
-var K3s = RKE2.WithConfigPath("/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl")
+var K3s = RKE2.WithConfigPath("/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl").
+	WithRestarter(containerd.K3sRestarter{})
 
 var K0s = Default.WithConfigPath("/etc/k0s/containerd.d/config.toml").
+	WithRestarter(containerd.K0sRestarter{}).
 	WithSetup(func(env Env) error {
 		_, err := env.HostFs.Stat(env.ConfigPath)
 		if err == nil {