diff --git a/.github/workflows/helm-chart-smoketest.yml b/.github/workflows/helm-chart-smoketest.yml index 3d451a1b..1021898f 100644 --- a/.github/workflows/helm-chart-smoketest.yml +++ b/.github/workflows/helm-chart-smoketest.yml @@ -108,10 +108,12 @@ jobs: - name: Create minikube cluster if: matrix.config.type == 'minikube' - uses: medyagh/setup-minikube@v0.0.19 - with: - container-runtime: containerd - kubernetes-version: ${{ env.K8S_VERSION }} + run: | + curl -LO https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64 + install minikube-linux-amd64 /usr/local/bin/minikube && rm minikube-linux-amd64 + docker build -t minikube-custom:v0.0.46-fixed -f ./images/test/Dockerfile.minikube-custom . --load + minikube start -p minikube --driver=docker --container-runtime=containerd --base-image="minikube-custom:v0.0.46-fixed" --kubernetes-version=${{ env.K8S_VERSION }} + kubectl wait pod --all --for=condition=Ready --namespace=kube-system --timeout=180s - name: Create microk8s cluster if: matrix.config.type == 'microk8s' @@ -150,10 +152,18 @@ jobs: - name: apply Spin shim run: | + shim_file=config/samples/test_shim_spin.yaml + if [[ "${{ matrix.config.type }}" == "microk8s" ]]; then + cp $shim_file config/samples/test_shim_spin_microk8s.yaml + shim_file=config/samples/test_shim_spin_microk8s.yaml + # update file to remove the 'containerdRuntimeOptions' field + # as there is a known bug that MicroK8s containerd does not pass the options + yq -i 'del(.spec.containerdRuntimeOptions)' $shim_file + fi # Ensure shim binary is compatible with runner arch yq -i '.spec.fetchStrategy.anonHttp.location = "https://github.com/spinframework/containerd-shim-spin/releases/download/${{ env.SHIM_SPIN_VERSION }}/containerd-shim-spin-v2-linux-x86_64.tar.gz"' \ - config/samples/test_shim_spin.yaml - kubectl apply -f config/samples/test_shim_spin.yaml + $shim_file + kubectl apply -f $shim_file - name: label nodes run: kubectl label node --all spin=true diff --git a/api/v1alpha1/shim_types.go b/api/v1alpha1/shim_types.go index bed349be..5b0805ea 100644 --- a/api/v1alpha1/shim_types.go +++ b/api/v1alpha1/shim_types.go @@ -26,6 +26,9 @@ type ShimSpec struct { FetchStrategy FetchStrategy `json:"fetchStrategy"` RuntimeClass RuntimeClassSpec `json:"runtimeClass"` RolloutStrategy RolloutStrategy `json:"rolloutStrategy"` + // ContainerdRuntimeOptions is a map of containerd runtime options for the shim plugin. + // See an example of configuring cgroup driver via runtime options: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver + ContainerdRuntimeOptions map[string]string `json:"containerdRuntimeOptions,omitempty"` } type FetchStrategy struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f2f2411a..f8234d24 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -174,6 +174,13 @@ func (in *ShimSpec) DeepCopyInto(out *ShimSpec) { out.FetchStrategy = in.FetchStrategy out.RuntimeClass = in.RuntimeClass out.RolloutStrategy = in.RolloutStrategy + if in.ContainerdRuntimeOptions != nil { + in, out := &in.ContainerdRuntimeOptions, &out.ContainerdRuntimeOptions + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShimSpec. diff --git a/cmd/node-installer/config.go b/cmd/node-installer/config.go index d7d3dc3c..e96809e9 100644 --- a/cmd/node-installer/config.go +++ b/cmd/node-installer/config.go @@ -20,6 +20,10 @@ type Config struct { Runtime struct { Name string ConfigPath string + // Options is a map of containerd runtime options for the shim plugin. + // See an example of the cgroup drive option here: + // https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver + Options map[string]string } RCM struct { Path string diff --git a/cmd/node-installer/detect_test.go b/cmd/node-installer/detect_test.go index 3c9ee05f..ee7ccbb4 100644 --- a/cmd/node-installer/detect_test.go +++ b/cmd/node-installer/detect_test.go @@ -45,7 +45,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", preset.MicroK8s.ConfigPath}, + Options map[string]string + }{"containerd", preset.MicroK8s.ConfigPath, nil}, struct { Path string AssetPath string @@ -64,7 +65,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", "/etc/containerd/not_found.toml"}, + Options map[string]string + }{"containerd", "/etc/containerd/not_found.toml", nil}, struct { Path string AssetPath string @@ -83,7 +85,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string @@ -102,7 +105,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string @@ -121,7 +125,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string @@ -140,7 +145,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string @@ -159,7 +165,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string @@ -178,7 +185,8 @@ func Test_DetectDistro(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", ""}, + Options map[string]string + }{"containerd", "", nil}, struct { Path string AssetPath string diff --git a/cmd/node-installer/install.go b/cmd/node-installer/install.go index 26b7c880..ab2e7fa0 100644 --- a/cmd/node-installer/install.go +++ b/cmd/node-installer/install.go @@ -17,6 +17,7 @@ package main import ( + "encoding/json" "fmt" "io/fs" "log/slog" @@ -50,6 +51,12 @@ var installCmd = &cobra.Command{ os.Exit(1) } + config.Runtime.Options, err = RuntimeOptions() + if err != nil { + slog.Error("failed to get runtime options", "error", err) + os.Exit(1) + } + if err := RunInstall(config, rootFs, hostFs, distro.Restarter); err != nil { slog.Error("failed to install", "error", err) os.Exit(1) @@ -82,7 +89,7 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res config.RCM.AssetPath = path.Dir(config.RCM.AssetPath) } - containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter) + containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter, config.Runtime.Options) shimConfig := shim.NewConfig(rootFs, hostFs, config.RCM.AssetPath, config.RCM.Path) anythingChanged := false @@ -109,6 +116,14 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res return nil } + // Ensure D-Bus is installed and running if using systemd + if _, err := containerd.ListSystemdUnits(); err == nil { + err = containerd.InstallDbus() + if err != nil { + return fmt.Errorf("failed to install D-Bus: %w", err) + } + } + slog.Info("restarting containerd") err = containerdConfig.RestartRuntime() if err != nil { @@ -117,3 +132,16 @@ func RunInstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.Res return nil } + +func RuntimeOptions() (map[string]string, error) { + runtimeOptions := make(map[string]string) + optionsJSON := os.Getenv("RUNTIME_OPTIONS") + config.Runtime.Options = make(map[string]string) + if optionsJSON != "" { + err := json.Unmarshal([]byte(optionsJSON), &runtimeOptions) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal runtime options JSON %s: %w", optionsJSON, err) + } + } + return runtimeOptions, nil +} diff --git a/cmd/node-installer/install_test.go b/cmd/node-installer/install_test.go index 714b30a4..1cce0730 100644 --- a/cmd/node-installer/install_test.go +++ b/cmd/node-installer/install_test.go @@ -49,7 +49,8 @@ func Test_RunInstall(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", "/etc/containerd/config.toml"}, + Options map[string]string + }{"containerd", "/etc/containerd/config.toml", nil}, struct { Path string AssetPath string @@ -68,7 +69,8 @@ func Test_RunInstall(t *testing.T) { struct { Name string ConfigPath string - }{"containerd", "/etc/containerd/config.toml"}, + Options map[string]string + }{"containerd", "/etc/containerd/config.toml", nil}, struct { Path string AssetPath string @@ -80,6 +82,27 @@ func Test_RunInstall(t *testing.T) { }, false, }, + { + // TODO figure out how to test that the runtime options are set in the config + "new shim with runtime options", + args{ + main.Config{ + struct { + Name string + ConfigPath string + Options map[string]string + }{"containerd", "/etc/containerd/config.toml", map[string]string{"SystemdCgroup": "true"}}, + struct { + Path string + AssetPath string + }{"/opt/rcm", "/assets"}, + struct{ RootPath string }{"/containerd/missing-containerd-shim-config"}, + }, + tests.FixtureFs("../../testdata/node-installer"), + tests.FixtureFs("../../testdata/node-installer/containerd/missing-containerd-shim-config"), + }, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/node-installer/uninstall.go b/cmd/node-installer/uninstall.go index 6772148c..f63d790b 100644 --- a/cmd/node-installer/uninstall.go +++ b/cmd/node-installer/uninstall.go @@ -45,6 +45,12 @@ var uninstallCmd = &cobra.Command{ config.Runtime.ConfigPath = distro.ConfigPath + config.Runtime.Options, err = RuntimeOptions() + if err != nil { + slog.Error("failed to get runtime options", "error", err) + os.Exit(1) + } + if err := RunUninstall(config, rootFs, hostFs, distro.Restarter); err != nil { slog.Error("failed to uninstall", "error", err) os.Exit(1) @@ -61,7 +67,7 @@ func RunUninstall(config Config, rootFs, hostFs afero.Fs, restarter containerd.R shimName := config.Runtime.Name runtimeName := path.Join(config.RCM.Path, "bin", shimName) - containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter) + containerdConfig := containerd.NewConfig(hostFs, config.Runtime.ConfigPath, restarter, config.Runtime.Options) shimConfig := shim.NewConfig(rootFs, hostFs, config.RCM.AssetPath, config.RCM.Path) binPath, err := shimConfig.Uninstall(shimName) diff --git a/config/crd/bases/runtime.spinkube.dev_shims.yaml b/config/crd/bases/runtime.spinkube.dev_shims.yaml index 1ecf6a15..7dff4377 100644 --- a/config/crd/bases/runtime.spinkube.dev_shims.yaml +++ b/config/crd/bases/runtime.spinkube.dev_shims.yaml @@ -49,6 +49,13 @@ spec: spec: description: ShimSpec defines the desired state of Shim properties: + containerdRuntimeOptions: + additionalProperties: + type: string + description: |- + ContainerdRuntimeOptions is a map of containerd runtime options for the shim plugin. + See an example of configuring cgroup driver via runtime options: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver + type: object fetchStrategy: properties: anonHttp: diff --git a/config/samples/test_shim_spin.yaml b/config/samples/test_shim_spin.yaml index 442b74d6..c4c9415d 100644 --- a/config/samples/test_shim_spin.yaml +++ b/config/samples/test_shim_spin.yaml @@ -17,6 +17,14 @@ spec: anonHttp: location: "https://github.com/spinframework/containerd-shim-spin/releases/download/v0.19.0/containerd-shim-spin-v2-linux-aarch64.tar.gz" + # Each runtime can provide a set of containerd runtime options to be set in the containerd + # configuration file. + containerdRuntimeOptions: + # The following option to pass cgroup driver information is available to runwasi based runtimes. + # For runwasi, the default cgroup driver is cgroupfs. Failure to configure the correct cgroup + # driver for runwasi shims may result in pod metrics failing to propagate accurately. + SystemdCgroup: "true" + runtimeClass: # Note: this name is used by the Spin Operator project as its default: # https://github.com/spinframework/spin-operator/blob/main/config/samples/spin-shim-executor.yaml diff --git a/deploy/helm/crds/runtime.spinkube.dev_shims.yaml b/deploy/helm/crds/runtime.spinkube.dev_shims.yaml index efd63ce4..7dff4377 100644 --- a/deploy/helm/crds/runtime.spinkube.dev_shims.yaml +++ b/deploy/helm/crds/runtime.spinkube.dev_shims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.16.3 name: shims.runtime.spinkube.dev spec: group: runtime.spinkube.dev @@ -30,20 +30,32 @@ spec: description: Shim is the Schema for the shims API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: ShimSpec defines the desired state of Shim properties: + containerdRuntimeOptions: + additionalProperties: + type: string + description: |- + ContainerdRuntimeOptions is a map of containerd runtime options for the shim plugin. + See an example of configuring cgroup driver via runtime options: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#cgroup-driver + type: object fetchStrategy: properties: anonHttp: @@ -100,43 +112,35 @@ spec: properties: conditions: items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -151,10 +155,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/images/test/Dockerfile.minikube-custom b/images/test/Dockerfile.minikube-custom new file mode 100644 index 00000000..f3d21f82 --- /dev/null +++ b/images/test/Dockerfile.minikube-custom @@ -0,0 +1,15 @@ +# The upstream minikube container is using expired OpenSUSE keys. +# This fetches the updated keys to enable updating packages. +# This is copied from the containerd-shim-spin project: https://github.com/spinframework/containerd-shim-spin/pull/289/commits/cc2e3de2a38935b50940b909707ffcaf42d3769e +FROM gcr.io/k8s-minikube/kicbase:v0.0.46@sha256:fd2d445ddcc33ebc5c6b68a17e6219ea207ce63c005095ea1525296da2d1a279 + +RUN apt-get update -y || true && \ + apt-get -y install wget curl apt-transport-https ca-certificates gnupg2 && \ + # Remove existing repository configurations to avoid conflicts + rm -f /etc/apt/sources.list.d/devel:kubic:*.list && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL "https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_22.04/Release.key" | gpg --dearmor > /etc/apt/keyrings/libcontainers-stable.gpg && \ + curl -fsSL "https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_22.04/Release.key" | gpg --dearmor > /etc/apt/keyrings/crio-stable.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/libcontainers-stable.gpg] https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_22.04/ /" > /etc/apt/sources.list.d/devel-kubic-libcontainers-stable.list && \ + echo "deb [signed-by=/etc/apt/keyrings/crio-stable.gpg] https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_22.04/ /" > /etc/apt/sources.list.d/devel-kubic-libcontainers-crio-stable.list && \ + apt-get update -y || true \ No newline at end of file diff --git a/internal/containerd/configure.go b/internal/containerd/configure.go index 1a050ff1..620960d8 100644 --- a/internal/containerd/configure.go +++ b/internal/containerd/configure.go @@ -32,16 +32,18 @@ type Restarter interface { } type Config struct { - hostFs afero.Fs - configPath string - restarter Restarter + hostFs afero.Fs + configPath string + restarter Restarter + runtimeOptions map[string]string } -func NewConfig(hostFs afero.Fs, configPath string, restarter Restarter) *Config { +func NewConfig(hostFs afero.Fs, configPath string, restarter Restarter, runtimeOptions map[string]string) *Config { return &Config{ - hostFs: hostFs, - configPath: configPath, - restarter: restarter, + hostFs: hostFs, + configPath: configPath, + restarter: restarter, + runtimeOptions: runtimeOptions, } } @@ -61,7 +63,7 @@ func (c *Config) AddRuntime(shimPath string) error { return nil } - cfg := generateConfig(shimPath, runtimeName, data) + cfg := generateConfig(shimPath, runtimeName, c.runtimeOptions, data) // Open file in append mode file, err := c.hostFs.OpenFile(c.configPath, os.O_APPEND|os.O_WRONLY, 0o644) //nolint:mnd // file permissions @@ -95,7 +97,7 @@ func (c *Config) RemoveRuntime(shimPath string) (changed bool, err error) { return false, nil } - cfg := generateConfig(shimPath, runtimeName, data) + cfg := generateConfig(shimPath, runtimeName, c.runtimeOptions, data) // Convert the file data to a string and replace the target string with an empty string. modifiedData := strings.ReplaceAll(string(data), cfg, "") @@ -113,7 +115,7 @@ func (c *Config) RestartRuntime() error { return c.restarter.Restart() } -func generateConfig(shimPath string, runtimeName string, configData []byte) string { +func generateConfig(shimPath string, runtimeName string, runtimeOptions map[string]string, configData []byte) string { // Config domain for containerd 1.0 (config version 2) domain := "io.containerd.grpc.v1.cri" if strings.Contains(string(configData), "version = 3") { @@ -121,9 +123,19 @@ func generateConfig(shimPath string, runtimeName string, configData []byte) stri domain = "io.containerd.cri.v1.runtime" } - return fmt.Sprintf(` + runtimeConfiguration := fmt.Sprintf(` # RCM runtime config for %s [plugins."%s".containerd.runtimes.%s] runtime_type = "%s" `, runtimeName, domain, runtimeName, shimPath) + // Add runtime options if any are provided + if len(runtimeOptions) > 0 { + options := fmt.Sprintf(`[plugins."%s".containerd.runtimes.%s.options]`, domain, runtimeName) + for k, v := range runtimeOptions { + options += fmt.Sprintf(` +%s = %s`, k, v) + } + runtimeConfiguration += options + } + return runtimeConfiguration } diff --git a/internal/containerd/configure_test.go b/internal/containerd/configure_test.go index dbc0921f..60173e9b 100644 --- a/internal/containerd/configure_test.go +++ b/internal/containerd/configure_test.go @@ -118,6 +118,49 @@ runtime_type = "/opt/rcm/bin/containerd-shim-spin-v1" } } +func TestConfig_AddRuntimeOptions(t *testing.T) { + wantFileContent := `[plugins] + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = 0 + startup_delay = "100ms" + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = true + [plugins."io.containerd.service.v1.tasks-service"] + blockio_config_file = "" + rdt_config_file = "" + +# RCM runtime config for spin-v1 +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v1] +runtime_type = "/opt/rcm/bin/containerd-shim-spin-v1" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin-v1.options] +SystemdCgroup = true` + t.Run("plugin options added", func(t *testing.T) { + c := &Config{ + hostFs: tests.FixtureFs("../../testdata/node-installer/containerd/missing-containerd-shim-config"), + configPath: "/etc/containerd/config.toml", + runtimeOptions: map[string]string{ + "SystemdCgroup": "true", + }, + } + err := c.AddRuntime("/opt/rcm/bin/containerd-shim-spin-v1") + + require.NoError(t, err) + + gotContent, err := afero.ReadFile(c.hostFs, c.configPath) + require.NoError(t, err) + + assert.Equal(t, wantFileContent, string(gotContent)) + }) +} + func TestConfig_RemoveRuntime(t *testing.T) { type fields struct { hostFs afero.Fs diff --git a/internal/containerd/install_dbus.go b/internal/containerd/install_dbus.go new file mode 100644 index 00000000..13cbdd26 --- /dev/null +++ b/internal/containerd/install_dbus.go @@ -0,0 +1,59 @@ +package containerd + +import ( + "fmt" + "log/slog" +) + +// InstallDbus checks if D-Bus service is installed and active. If not, installs D-Bus +// and starts the service. +// NOTE: this limits support to systems using systemctl to manage systemd. +func InstallDbus() error { + cmd := nsenterCmd("systemctl", "start", "dbus", "--quiet") + if err := cmd.Run(); err == nil { + slog.Info("D-Bus is already installed and running") + return nil + } + slog.Info("installing D-Bus") + + type pkgManager struct { + name string + check []string + update []string + install []string + } + + managers := []pkgManager{ + {"apt-get", []string{"which", "apt-get"}, []string{"apt-get", "update", "--yes"}, []string{"apt-get", "install", "--yes", "dbus"}}, + {"dnf", []string{"which", "dnf"}, []string{}, []string{"dnf", "install", "--yes", "dbus"}}, + {"apk", []string{"which", "apk"}, []string{}, []string{"apk", "add", "dbus"}}, + {"yum", []string{"which", "yum"}, []string{}, []string{"yum", "install", "--yes", "dbus"}}, + } + installed := false + for _, mgr := range managers { + if err := nsenterCmd(mgr.check...).Run(); err == nil { + if len(mgr.update) != 0 { + if err := nsenterCmd(mgr.update...).Run(); err != nil { + return fmt.Errorf("failed to update package manager %s: %w", mgr.name, err) + } + } + if err := nsenterCmd(mgr.install...).Run(); err != nil { + return fmt.Errorf("failed to install D-Bus with %s: %w", mgr.name, err) + } + installed = true + break + } + } + + if !installed { + return fmt.Errorf("could not install D-Bus as no supported package manager found") + } + + slog.Info("restarting D-Bus") + cmd = nsenterCmd("systemctl", "restart", "dbus") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to restart D-Bus: %w", err) + } + + return nil +} diff --git a/internal/containerd/restart_unix.go b/internal/containerd/restart_unix.go index ffe2550a..ed441d45 100644 --- a/internal/containerd/restart_unix.go +++ b/internal/containerd/restart_unix.go @@ -40,7 +40,7 @@ func NewDefaultRestarter() Restarter { func (c defaultRestarter) Restart() error { // If listing systemd units succeeds, prefer systemctl restart; otherwise kill pid - if _, err := listSystemdUnits(); err == nil { + if _, err := ListSystemdUnits(); err == nil { out, err := nsenterCmd("systemctl", "restart", "containerd").CombinedOutput() slog.Debug(string(out)) if err != nil { @@ -67,7 +67,7 @@ type K0sRestarter struct{} func (c K0sRestarter) Restart() error { // First, collect systemd units to determine which mode k0s is running in, eg // k0sworker or k0scontroller - units, err := listSystemdUnits() + units, err := ListSystemdUnits() if err != nil { return fmt.Errorf("unable to list systemd units: %w", err) } @@ -88,7 +88,7 @@ func (c K3sRestarter) Restart() error { // This restarter will be used both for stock K3s distros, which use systemd as well as K3d, which does not. // If listing systemd units succeeds, prefer systemctl restart; otherwise kill pid - if _, err := listSystemdUnits(); err == nil { + if _, err := ListSystemdUnits(); err == nil { out, err := nsenterCmd("systemctl", "restart", "k3s").CombinedOutput() slog.Debug(string(out)) if err != nil { @@ -130,7 +130,7 @@ type RKE2Restarter struct{} func (c RKE2Restarter) Restart() error { // First, collect systemd units to determine which mode rke2 is running in, eg // rke2-agent or rke2-server - units, err := listSystemdUnits() + units, err := ListSystemdUnits() if err != nil { return fmt.Errorf("unable to list systemd units: %w", err) } @@ -145,7 +145,7 @@ func (c RKE2Restarter) Restart() error { return nil } -func listSystemdUnits() ([]byte, error) { +func ListSystemdUnits() ([]byte, error) { return nsenterCmd("systemctl", "list-units", "--type", "service").CombinedOutput() } diff --git a/internal/controller/shim_controller.go b/internal/controller/shim_controller.go index 69a58f5d..83bf520f 100644 --- a/internal/controller/shim_controller.go +++ b/internal/controller/shim_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "encoding/json" "errors" "fmt" "math" @@ -379,6 +380,8 @@ func (sr *ShimReconciler) setOperationConfiguration(shim *rcmv1.Shim, opConfig * } // createJobManifest creates a Job manifest for a Shim. +// +//nolint:funlen // function is longer due to scaffolding an entire K8s Job manifest func (sr *ShimReconciler) createJobManifest(shim *rcmv1.Shim, node *corev1.Node, operation string) (*batchv1.Job, error) { opConfig := opConfig{ operation: operation, @@ -457,6 +460,19 @@ func (sr *ShimReconciler) createJobManifest(shim *rcmv1.Shim, node *corev1.Node, }, }, } + + if shim.Spec.ContainerdRuntimeOptions != nil { + optionsJSON, err := json.Marshal(shim.Spec.ContainerdRuntimeOptions) + if err != nil { + log.Error().Msgf("Unable to marshal runtime options: %s", err) + } else { + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "RUNTIME_OPTIONS", + Value: string(optionsJSON), + }) + } + } + // set ttl for the installer job only if specified by the user if ttlStr := os.Getenv("SHIM_NODE_INSTALLER_JOB_TTL"); ttlStr != "" { if ttl, err := strconv.ParseInt(ttlStr, 10, 32); err == nil && ttl > 0 {