Move cri base plugin to CRI runtime service

Create new plugin type for CRI runtime and image services.

Signed-off-by: Derek McGowan <[email protected]>

Author: dmcgowan

cmd/containerd/builtins/cri.go

@@ -21,4 +21,5 @@ package builtins
 import (
 	_ "github.com/containerd/containerd/v2/pkg/cri"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 )

contrib/fuzz/builtins.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/containerd/containerd/v2/pkg/events/plugin"
 	_ "github.com/containerd/containerd/v2/pkg/nri/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 	_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/gc"
 	_ "github.com/containerd/containerd/v2/plugins/imageverifier"

contrib/fuzz/cri_server_fuzzer.go

@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/containerd/v2/pkg/cri/server"
 	"github.com/containerd/containerd/v2/pkg/cri/server/images"
 	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/errdefs"
 )
 
 func FuzzCRIServer(data []byte) int {
@@ -42,7 +43,6 @@ func FuzzCRIServer(data []byte) int {
 	}
 	defer client.Close()
 
-	config := criconfig.Config{}
 	imageConfig := criconfig.ImageConfig{}
 
 	imageService, err := images.NewService(imageConfig, &images.CRIImageServiceOptions{
@@ -52,10 +52,10 @@ func FuzzCRIServer(data []byte) int {
 		panic(err)
 	}
 
-	c, rs, err := server.NewCRIService(config, &server.CRIServiceOptions{
-		ImageService: imageService,
-		Client:       client,
-		BaseOCISpecs: map[string]*oci.Spec{},
+	c, rs, err := server.NewCRIService(&server.CRIServiceOptions{
+		RuntimeService: &fakeRuntimeService{},
+		ImageService:   imageService,
+		Client:         client,
 	})
 	if err != nil {
 		panic(err)
@@ -68,6 +68,16 @@ func FuzzCRIServer(data []byte) int {
 	})
 }
 
+type fakeRuntimeService struct{}
+
+func (fakeRuntimeService) Config() criconfig.Config {
+	return criconfig.Config{}
+}
+
+func (fakeRuntimeService) LoadOCISpec(string) (*oci.Spec, error) {
+	return nil, errdefs.ErrNotFound
+}
+
 type service struct {
 	server.CRIService
 	runtime.RuntimeServiceServer

integration/build_local_containerd_helper_test.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/containerd/containerd/v2/core/runtime/v2/runc/options"
 	_ "github.com/containerd/containerd/v2/pkg/events/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 	_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/gc"
 	_ "github.com/containerd/containerd/v2/plugins/leases"

pkg/cri/config/config.go

@@ -319,8 +319,6 @@ type PluginConfig struct {
 	ContainerdConfig `toml:"containerd" json:"containerd"`
 	// CniConfig contains config related to cni
 	CniConfig `toml:"cni" json:"cni"`
-	// DisableTCPService disables serving CRI on the TCP server.
-	DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
 	// StreamServerAddress is the ip address streaming server is listening on.
 	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
 	// StreamServerPort is the port streaming server is listening on.
@@ -433,6 +431,12 @@ type Config struct {
 	StateDir string `json:"stateDir"`
 }
 
+// ServiceConfig contains all the configuration for the CRI API server.
+type ServiceConfig struct {
+	// DisableTCPService disables serving CRI on the TCP server.
+	DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
+}
+
 const (
 	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
 	RuntimeUntrusted = "untrusted"

pkg/cri/config/config_unix.go

@@ -89,7 +89,6 @@ func DefaultConfig() PluginConfig {
 				},
 			},
 		},
-		DisableTCPService:    true,
 		StreamServerAddress:  "127.0.0.1",
 		StreamServerPort:     "0",
 		StreamIdleTimeout:    streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour

pkg/cri/config/config_windows.go

@@ -78,7 +78,6 @@ func DefaultConfig() PluginConfig {
 				},
 			},
 		},
-		DisableTCPService:   true,
 		StreamServerAddress: "127.0.0.1",
 		StreamServerPort:    "0",
 		StreamIdleTimeout:   streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour

pkg/cri/cri.go

@@ -17,6 +17,7 @@
 package cri
 
 import (
+	"context"
 	"fmt"
 	"io"
 
@@ -25,13 +26,13 @@ import (
 	"github.com/containerd/plugin/registry"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	srvconfig "github.com/containerd/containerd/v2/cmd/containerd/server/config"
 	"github.com/containerd/containerd/v2/core/sandbox"
 	criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
 	"github.com/containerd/containerd/v2/pkg/cri/constants"
 	"github.com/containerd/containerd/v2/pkg/cri/instrument"
 	"github.com/containerd/containerd/v2/pkg/cri/nri"
 	"github.com/containerd/containerd/v2/pkg/cri/server"
-	"github.com/containerd/containerd/v2/pkg/cri/server/base"
 	nriservice "github.com/containerd/containerd/v2/pkg/nri"
 	"github.com/containerd/containerd/v2/plugins"
 	"github.com/containerd/platforms"
@@ -43,13 +44,11 @@ import (
 
 // Register CRI service plugin
 func init() {
-
 	registry.Register(&plugin.Registration{
 		Type: plugins.GRPCPlugin,
 		ID:   "cri",
 		Requires: []plugin.Type{
-			plugins.CRIImagePlugin,
-			plugins.InternalPlugin,
+			plugins.CRIServicePlugin,
 			plugins.SandboxControllerPlugin,
 			plugins.NRIApiPlugin,
 			plugins.EventPlugin,
@@ -58,23 +57,46 @@ func init() {
 			plugins.SandboxStorePlugin,
 			plugins.TransferPlugin,
 		},
+		Config: &criconfig.ServiceConfig{
+			DisableTCPService: true,
+		},
+		ConfigMigration: func(ctx context.Context, version int, pluginConfigs map[string]interface{}) error {
+			if version >= srvconfig.CurrentConfigVersion {
+				return nil
+			}
+			const pluginName = string(plugins.GRPCPlugin) + ".cri"
+			original, ok := pluginConfigs[pluginName]
+			if !ok {
+				return nil
+			}
+			src := original.(map[string]interface{})
+
+			// Currently only a single key migrated
+			if val, ok := src["disable_tcp_service"]; ok {
+				pluginConfigs[pluginName] = map[string]interface{}{
+					"disable_tcp_service": val,
+				}
+			} else {
+				delete(pluginConfigs, pluginName)
+			}
+			return nil
+		},
 		InitFn: initCRIService,
 	})
 }
 
 func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	ctx := ic.Context
+	config := ic.Config.(*criconfig.ServiceConfig)
 
-	// Get base CRI dependencies.
-	criBasePlugin, err := ic.GetByID(plugins.InternalPlugin, "cri")
+	// Get runtime service.
+	criRuntimePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "runtime")
 	if err != nil {
-		return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err)
+		return nil, fmt.Errorf("unable to load CRI runtime service plugin dependency: %w", err)
 	}
-	criBase := criBasePlugin.(*base.CRIBase)
-	c := criBase.Config
 
 	// Get image service.
-	criImagePlugin, err := ic.GetSingle(plugins.CRIImagePlugin)
+	criImagePlugin, err := ic.GetByID(plugins.CRIServicePlugin, "images")
 	if err != nil {
 		return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err)
 	}
@@ -98,15 +120,16 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	}
 
 	options := &server.CRIServiceOptions{
+		RuntimeService:     criRuntimePlugin.(server.RuntimeService),
 		ImageService:       criImagePlugin.(server.ImageService),
 		NRI:                getNRIAPI(ic),
 		Client:             client,
 		SandboxControllers: sbControllers,
-		BaseOCISpecs:       criBase.BaseOCISpecs,
 	}
 	is := criImagePlugin.(imageService).GRPCService()
 
-	s, rs, err := server.NewCRIService(criBase.Config, options)
+	// TODO: More options specifically for grpc service?
+	s, rs, err := server.NewCRIService(options)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create CRI service: %w", err)
 	}
@@ -127,7 +150,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 		initializer:          s,
 	}
 
-	if c.DisableTCPService {
+	if config.DisableTCPService {
 		return service, nil
 	}
 

pkg/cri/server/container_create.go

@@ -394,9 +394,9 @@ func (c *criService) runtimeSpec(id string, platform platforms.Platform, baseSpe
 	container := &containers.Container{ID: id}
 
 	if baseSpecFile != "" {
-		baseSpec, ok := c.baseOCISpecs[baseSpecFile]
-		if !ok {
-			return nil, fmt.Errorf("can't find base OCI spec %q", baseSpecFile)
+		baseSpec, err := c.LoadOCISpec(baseSpecFile)
+		if err != nil {
+			return nil, fmt.Errorf("can't load base OCI spec %q: %w", baseSpecFile, err)
 		}
 
 		spec := oci.Spec{}

pkg/cri/server/container_create_linux_test.go

@@ -1680,23 +1680,24 @@ func TestPrivilegedDevices(t *testing.T) {
 }
 
 func TestBaseOCISpec(t *testing.T) {
-	c := newTestCRIService()
 	baseLimit := int64(100)
-	c.baseOCISpecs = map[string]*oci.Spec{
-		"/etc/containerd/cri-base.json": {
-			Process: &runtimespec.Process{
-				User: runtimespec.User{AdditionalGids: []uint32{9999}},
-				Capabilities: &runtimespec.LinuxCapabilities{
-					Permitted: []string{"CAP_SETUID"},
+	c := newTestCRIService(withRuntimeService(&fakeRuntimeService{
+		ocispecs: map[string]*oci.Spec{
+			"/etc/containerd/cri-base.json": {
+				Process: &runtimespec.Process{
+					User: runtimespec.User{AdditionalGids: []uint32{9999}},
+					Capabilities: &runtimespec.LinuxCapabilities{
+						Permitted: []string{"CAP_SETUID"},
+					},
 				},
-			},
-			Linux: &runtimespec.Linux{
-				Resources: &runtimespec.LinuxResources{
-					Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: &baseLimit}, // Will be overwritten by `getCreateContainerTestData`
+					},
 				},
 			},
 		},
-	}
+	}))
 
 	ociRuntime := config.Runtime{}
 	ociRuntime.BaseRuntimeSpec = "/etc/containerd/cri-base.json"