Enable configuring containerd plugin runtime options

.github/workflows/helm-chart-smoketest.yml

@@ -108,10 +108,12 @@ jobs:
 
       - name: Create minikube cluster
         if: matrix.config.type == 'minikube'
-        uses: medyagh/[email protected]
-        with:
-          container-runtime: containerd
-          kubernetes-version: ${{ env.K8S_VERSION }}
+        run: |
+          curl -LO https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64
+          install minikube-linux-amd64 /usr/local/bin/minikube && rm minikube-linux-amd64
+          docker build -t minikube-custom:v0.0.46-fixed -f ./images/test/Dockerfile.minikube-custom . --load
+          minikube start -p minikube --driver=docker --container-runtime=containerd --base-image="minikube-custom:v0.0.46-fixed" --kubernetes-version=${{ env.K8S_VERSION }}
+          kubectl wait pod --all --for=condition=Ready --namespace=kube-system --timeout=180s
 
       - name: Create microk8s cluster
         if: matrix.config.type == 'microk8s'
@@ -150,10 +152,18 @@ jobs:
 
       - name: apply Spin shim
         run: |
+          shim_file=config/samples/test_shim_spin.yaml
+          if [[ "${{ matrix.config.type }}" == "microk8s" ]]; then
+            cp $shim_file config/samples/test_shim_spin_microk8s.yaml
+            shim_file=config/samples/test_shim_spin_microk8s.yaml
+            # update file to remove the 'containerdRuntimeOptions' field
+            # as there is a known bug that MicroK8s containerd does not pass the options
+            yq -i 'del(.spec.containerdRuntimeOptions)' $shim_file
+          fi
           # Ensure shim binary is compatible with runner arch
           yq -i '.spec.fetchStrategy.anonHttp.location = "https://github.com/spinframework/containerd-shim-spin/releases/download/${{ env.SHIM_SPIN_VERSION }}/containerd-shim-spin-v2-linux-x86_64.tar.gz"' \
-            config/samples/test_shim_spin.yaml
-          kubectl apply -f config/samples/test_shim_spin.yaml
+            $shim_file
+          kubectl apply -f $shim_file
 
       - name: label nodes
         run: kubectl label node --all spin=true

api/v1alpha1/shim_types.go

@@ -26,6 +26,9 @@ type ShimSpec struct {
 	FetchStrategy   FetchStrategy     `json:"fetchStrategy"`
 	RuntimeClass    RuntimeClassSpec  `json:"runtimeClass"`
 	RolloutStrategy RolloutStrategy   `json:"rolloutStrategy"`
+	// ContainerdRuntimeOptions is a map of containerd runtime options for the shim plugin.
+	// See an example of configuring cgroup driver via runtime options: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver
+	ContainerdRuntimeOptions map[string]string `json:"containerdRuntimeOptions,omitempty"`
 }
 
 type FetchStrategy struct {

cmd/node-installer/config.go

@@ -20,6 +20,10 @@ type Config struct {
 	Runtime struct {
 		Name       string
 		ConfigPath string
+		// Options is a map of containerd runtime options for the shim plugin.
+		// See an example of the cgroup drive option here:
+		// https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver
+		Options map[string]string
 	}
 	RCM struct {
 		Path      string

cmd/node-installer/detect_test.go

@@ -45,7 +45,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", preset.MicroK8s.ConfigPath},
+						Options    map[string]string
+					}{"containerd", preset.MicroK8s.ConfigPath, nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -64,7 +65,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", "/etc/containerd/not_found.toml"},
+						Options    map[string]string
+					}{"containerd", "/etc/containerd/not_found.toml", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -83,7 +85,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -102,7 +105,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -121,7 +125,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -140,7 +145,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -159,7 +165,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -178,7 +185,8 @@ func Test_DetectDistro(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", ""},
+						Options    map[string]string
+					}{"containerd", "", nil},
 					struct {
 						Path      string
 						AssetPath string

cmd/node-installer/install.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/fs"
 	"log/slog"
@@ -50,6 +51,12 @@ var installCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		config.Runtime.Options, err = RuntimeOptions()
+		if err != nil {
+			slog.Error("failed to get runtime options", "error", err)
+			os.Exit(1)
+		}
+
 		if err := RunInstall(config, rootFs, hostFs, distro.Restarter); err != nil {
 			slog.Error("failed to install", "error", err)
 			os.Exit(1)
@@ -82,7 +89,7 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res
 		config.RCM.AssetPath = path.Dir(config.RCM.AssetPath)
 	}
 
-	containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter)
+	containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter, config.Runtime.Options)
 	shimConfig := shim.NewConfig(rootFs, hostFs, config.RCM.AssetPath, config.RCM.Path)
 
 	anythingChanged := false
@@ -109,6 +116,14 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res
 		return nil
 	}
 
+	// Ensure D-Bus is installed and running if using systemd
+	if _, err := containerd.ListSystemdUnits(); err == nil {
+		err = containerd.InstallDbus()
+		if err != nil {
+			return fmt.Errorf("failed to install D-Bus: %w", err)
+		}
+	}
+
 	slog.Info("restarting containerd")
 	err = containerdConfig.RestartRuntime()
 	if err != nil {
@@ -117,3 +132,16 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res
 
 	return nil
 }
+
+func RuntimeOptions() (map[string]string, error) {
+	runtimeOptions := make(map[string]string)
+	optionsJSON := os.Getenv("RUNTIME_OPTIONS")
+	config.Runtime.Options = make(map[string]string)
+	if optionsJSON != "" {
+		err := json.Unmarshal([]byte(optionsJSON), &runtimeOptions)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal runtime options JSON %s: %w", optionsJSON, err)
+		}
+	}
+	return runtimeOptions, nil
+}

cmd/node-installer/install_test.go

@@ -49,7 +49,8 @@ func Test_RunInstall(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", "/etc/containerd/config.toml"},
+						Options    map[string]string
+					}{"containerd", "/etc/containerd/config.toml", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -68,7 +69,8 @@ func Test_RunInstall(t *testing.T) {
 					struct {
 						Name       string
 						ConfigPath string
-					}{"containerd", "/etc/containerd/config.toml"},
+						Options    map[string]string
+					}{"containerd", "/etc/containerd/config.toml", nil},
 					struct {
 						Path      string
 						AssetPath string
@@ -80,6 +82,27 @@ func Test_RunInstall(t *testing.T) {
 			},
 			false,
 		},
+		{
+			// TODO figure out how to test that the runtime options are set in the config
+			"new shim with runtime options",
+			args{
+				main.Config{
+					struct {
+						Name       string
+						ConfigPath string
+						Options    map[string]string
+					}{"containerd", "/etc/containerd/config.toml", map[string]string{"SystemdCgroup": "true"}},
+					struct {
+						Path      string
+						AssetPath string
+					}{"/opt/rcm", "/assets"},
+					struct{ RootPath string }{"/containerd/missing-containerd-shim-config"},
+				},
+				tests.FixtureFs("../../testdata/node-installer"),
+				tests.FixtureFs("../../testdata/node-installer/containerd/missing-containerd-shim-config"),
+			},
+			false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

cmd/node-installer/uninstall.go

@@ -45,6 +45,12 @@ var uninstallCmd = &cobra.Command{
 
 		config.Runtime.ConfigPath = distro.ConfigPath
 
+		config.Runtime.Options, err = RuntimeOptions()
+		if err != nil {
+			slog.Error("failed to get runtime options", "error", err)
+			os.Exit(1)
+		}
+
 		if err := RunUninstall(config, rootFs, hostFs, distro.Restarter); err != nil {
 			slog.Error("failed to uninstall", "error", err)
 			os.Exit(1)
@@ -61,7 +67,7 @@ func RunUninstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.R
 	shimName := config.Runtime.Name
 	runtimeName := path.Join(config.RCM.Path, "bin", shimName)
 
-	containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter)
+	containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter, config.Runtime.Options)
 	shimConfig := shim.NewConfig(rootFs, hostFs, config.RCM.AssetPath, config.RCM.Path)
 
 	binPath, err := shimConfig.Uninstall(shimName)

config/crd/bases/runtime.spinkube.dev_shims.yaml

@@ -49,6 +49,13 @@ spec:
           spec:
             description: ShimSpec defines the desired state of Shim
             properties:
+              containerdRuntimeOptions:
+                additionalProperties:
+                  type: string
+                description: |-
+                  ContainerdRuntimeOptions is a map of containerd runtime options for the shim plugin.
+                  See an example of configuring cgroup driver via runtime options: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver
+                type: object
               fetchStrategy:
                 properties:
                   anonHttp:

config/samples/test_shim_spin.yaml

@@ -17,6 +17,14 @@ spec:
     anonHttp:
       location: "https://github.com/spinframework/containerd-shim-spin/releases/download/v0.19.0/containerd-shim-spin-v2-linux-aarch64.tar.gz"
 
+  # Each runtime can provide a set of containerd runtime options to be set in the containerd
+  # configuration file.
+  containerdRuntimeOptions:
+    # The following option to pass cgroup driver information is available to runwasi based runtimes.
+    # For runwasi, the default cgroup driver is cgroupfs. Failure to configure the correct cgroup
+    # driver for runwasi shims may result in pod metrics failing to propagate accurately.
+    SystemdCgroup: "true"
+
   runtimeClass:
     # Note: this name is used by the Spin Operator project as its default:
     # https://github.com/spinframework/spin-operator/blob/main/config/samples/spin-shim-executor.yaml