exec: cdi device support

Signed-off-by: CrazyMax <[email protected]>

Author: crazy-max

client/client_test.go

@@ -274,6 +274,15 @@ func testIntegration(t *testing.T, funcs ...func(t *testing.T, sb integration.Sa
 			"dns": bridgeDNSNetwork,
 		}),
 	)
+
+	integration.Run(t, integration.TestFuncs(
+		testCDI,
+	),
+		mirrors,
+		integration.WithMatrix("cdi", map[string]interface{}{
+			"enabled": enableCDI,
+		}),
+	)
 }
 
 func newContainerd(cdAddress string) (*ctd.Client, error) {
@@ -7436,8 +7445,8 @@ func testMergeOp(t *testing.T, sb integration.Sandbox) {
 		File(llb.Mkfile("bar/D", 0644, []byte("D"))).
 		File(llb.Mkfile("bar/E", 0755, nil)).
 		File(llb.Mkfile("qaz", 0644, nil)),
-	// /foo from stateE is not here because it is deleted in stateB, which is part of a submerge of mergeD
 	)
+	// /foo from stateE is not here because it is deleted in stateB, which is part of a submerge of mergeD
 }
 
 func testMergeOpCacheInline(t *testing.T, sb integration.Sandbox) {
@@ -10986,3 +10995,79 @@ func (w warningsListOutput) String() string {
 	}
 	return b.String()
 }
+
+type cdiEnabled struct{}
+
+func (*cdiEnabled) UpdateConfigFile(in string) string {
+	return in + `
+[cdi]
+enabled = true
+`
+}
+
+var (
+	enableCDI integration.ConfigUpdater = &cdiEnabled{}
+)
+
+func testCDI(t *testing.T, sb integration.Sandbox) {
+	if sb.Rootless() {
+		t.SkipNow()
+	}
+
+	integration.SkipOnPlatform(t, "windows")
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(`
+cdiVersion: "0.3.0"
+kind: "vendor1.com/device"
+devices:
+- name: foo
+  containerEdits:
+    env:
+    - FOO=injected
+`), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor2-device.yaml"), []byte(`
+cdiVersion: "0.3.0"
+kind: "vendor2.com/device"
+devices:
+- name: bar
+  containerEdits:
+    env:
+    - BAR=injected
+`), 0600))
+
+	busybox := llb.Image("busybox:latest")
+	st := llb.Scratch()
+
+	run := func(cmd string, ro ...llb.RunOption) {
+		st = busybox.Run(append(ro, llb.Shlex(cmd), llb.Dir("/wd"))...).AddMount("/wd", st)
+	}
+
+	run(`sh -c 'env|sort | tee foo.env'`, llb.AddCDIDevice("vendor1.com/device=foo"))
+	run(`sh -c 'env|sort | tee bar.env'`, llb.AddCDIDevice("vendor2.com/device=bar"))
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "foo.env"))
+	require.NoError(t, err)
+	require.Contains(t, strings.TrimSpace(string(dt)), `FOO=injected`)
+
+	dt2, err := os.ReadFile(filepath.Join(destDir, "bar.env"))
+	require.NoError(t, err)
+	require.Contains(t, strings.TrimSpace(string(dt2)), `BAR=injected`)
+}

client/llb/exec.go

@@ -266,6 +266,22 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
 		Network:  network,
 		Security: security,
 	}
+
+	cdiDevices, err := getCDIDevice(e.base)(ctx, c)
+	if err != nil {
+		return "", nil, nil, nil, err
+	}
+	if len(cdiDevices) > 0 {
+		addCap(&e.constraints, pb.CapExecMetaCDI)
+		cd := make([]*pb.CDIDevice, len(cdiDevices))
+		for i, d := range cdiDevices {
+			cd[i] = &pb.CDIDevice{
+				Name: d.Name,
+			}
+		}
+		peo.CdiDevices = cd
+	}
+
 	if network != NetModeSandbox {
 		addCap(&e.constraints, pb.CapExecMetaNetwork)
 	}
@@ -624,6 +640,12 @@ func AddUlimit(name UlimitName, soft int64, hard int64) RunOption {
 	})
 }
 
+func AddCDIDevice(name string) RunOption {
+	return runOptionFunc(func(ei *ExecInfo) {
+		ei.State = ei.State.AddCDIDevice(name)
+	})
+}
+
 func ValidExitCodes(codes ...int) RunOption {
 	return runOptionFunc(func(ei *ExecInfo) {
 		ei.State = validExitCodes(codes...)(ei.State)

client/llb/meta.go

@@ -24,6 +24,7 @@ var (
 	keyExtraHost      = contextKeyT("llb.exec.extrahost")
 	keyHostname       = contextKeyT("llb.exec.hostname")
 	keyUlimit         = contextKeyT("llb.exec.ulimit")
+	keyDevice         = contextKeyT("llb.exec.device")
 	keyCgroupParent   = contextKeyT("llb.exec.cgroup.parent")
 	keyUser           = contextKeyT("llb.exec.user")
 	keyValidExitCodes = contextKeyT("llb.exec.validexitcodes")
@@ -305,6 +306,33 @@ func getUlimit(s State) func(context.Context, *Constraints) ([]*pb.Ulimit, error
 	}
 }
 
+func cdiDevice(name string) StateOption {
+	return func(s State) State {
+		return s.withValue(keyDevice, func(ctx context.Context, c *Constraints) (interface{}, error) {
+			v, err := getCDIDevice(s)(ctx, c)
+			if err != nil {
+				return nil, err
+			}
+			return append(v, &pb.CDIDevice{
+				Name: name,
+			}), nil
+		})
+	}
+}
+
+func getCDIDevice(s State) func(context.Context, *Constraints) ([]*pb.CDIDevice, error) {
+	return func(ctx context.Context, c *Constraints) ([]*pb.CDIDevice, error) {
+		v, err := s.getValue(keyDevice)(ctx, c)
+		if err != nil {
+			return nil, err
+		}
+		if v != nil {
+			return v.([]*pb.CDIDevice), nil
+		}
+		return nil, nil
+	}
+}
+
 func cgroupParent(cp string) StateOption {
 	return func(s State) State {
 		return s.WithValue(keyCgroupParent, cp)

client/llb/state.go

@@ -476,6 +476,12 @@ func (s State) AddUlimit(name UlimitName, soft int64, hard int64) State {
 	return ulimit(name, soft, hard)(s)
 }
 
+// AddCDIDevice sets the fully qualified CDI device name.
+// https://github.com/cncf-tags/container-device-interface/blob/main/SPEC.md
+func (s State) AddCDIDevice(name string) State {
+	return cdiDevice(name)(s)
+}
+
 // WithCgroupParent sets the parent cgroup for any containers created from this state.
 // This is useful when you want to apply resource constraints to a group of containers.
 // Cgroups are Linux specific and only applies to containers created from this state such as via `[State.Run]`

executor/executor.go

@@ -22,6 +22,7 @@ type Meta struct {
 	ReadonlyRootFS bool
 	ExtraHosts     []HostIP
 	Ulimit         []*pb.Ulimit
+	CDIDevices     []*pb.CDIDevice
 	CgroupParent   string
 	NetMode        pb.NetMode
 	SecurityMode   pb.SecurityMode

executor/oci/spec.go

@@ -110,6 +110,12 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 		return nil, nil, err
 	}
 
+	if cdiOpts, err := generateCDIOpts(ctx, meta.CDIDevices); err == nil {
+		opts = append(opts, cdiOpts...)
+	} else {
+		return nil, nil, err
+	}
+
 	hostname := defaultHostname
 	if meta.Hostname != "" {
 		hostname = meta.Hostname

executor/oci/spec_darwin.go

@@ -1,6 +1,8 @@
 package oci
 
 import (
+	"context"
+
 	"github.com/containerd/containerd/v2/core/mount"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	"github.com/containerd/continuity/fs"
@@ -62,3 +64,10 @@ func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
 	m.Source = src
 	return m, func() error { return nil }, nil
 }
+
+func generateCDIOpts(ctx context.Context, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	return nil, errors.New("no support for CDI on Darwin")
+}

executor/oci/spec_freebsd.go

@@ -1,6 +1,8 @@
 package oci
 
 import (
+	"context"
+
 	"github.com/containerd/containerd/v2/core/mount"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	"github.com/containerd/continuity/fs"
@@ -70,3 +72,10 @@ func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
 	m.Source = src
 	return m, func() error { return nil }, nil
 }
+
+func generateCDIOpts(ctx context.Context, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	return nil, errors.New("no support for CDI on FreeBSD")
+}

executor/oci/spec_linux.go

@@ -14,16 +14,20 @@ import (
 	"github.com/containerd/containerd/v2/pkg/oci"
 	cdseccomp "github.com/containerd/containerd/v2/pkg/seccomp"
 	"github.com/containerd/continuity/fs"
+	"github.com/containerd/log"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/profiles/seccomp"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/entitlements/security"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	selinux "github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+	"tags.cncf.io/container-device-interface/pkg/parser"
 )
 
 var (
@@ -148,6 +152,72 @@ func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) {
 	}, nil
 }
 
+// genereateCDIOptions creates the OCI runtime spec options for injecting CDI
+// devices. Two options are returned: The first ensures that the CDI registry
+// is initialized with refresh disabled, and the second injects the devices
+// into the container.
+func generateCDIOpts(ctx context.Context, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	var dd []string
+	for _, d := range devices {
+		if d == nil {
+			continue
+		}
+		if _, _, _, err := parser.ParseQualifiedName(d.Name); err != nil {
+			return nil, errors.Wrapf(err, "invalid CDI device name %s", d.Name)
+		}
+		dd = append(dd, d.Name)
+	}
+
+	// withStaticCDIRegistry inits the CDI registry and disables auto-refresh.
+	// This is used from the `run` command to avoid creating a registry with
+	// auto-refresh enabled. It also provides a way to override the CDI spec
+	// file paths if required.
+	withStaticCDIRegistry := func() oci.SpecOpts {
+		return func(ctx context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+			_ = cdi.Configure(cdi.WithAutoRefresh(false))
+			if err := cdi.Refresh(); err != nil {
+				// We don't consider registry refresh failure a fatal error.
+				// For instance, a dynamically generated invalid CDI Spec file
+				// for any particular vendor shouldn't prevent injection of
+				// devices of different vendors. CDI itself knows better, and
+				// it will fail the injection if necessary.
+				bklog.G(ctx).Warnf("CDI registry refresh failed: %v", err)
+			}
+			return nil
+		}
+	}
+
+	// withCDIDevices injects the requested CDI devices into the OCI specification.
+	// FIXME: Use oci.WithCDIDevices once we switch to containerd 2.0.
+	withCDIDevices := func(devices ...string) oci.SpecOpts {
+		return func(ctx context.Context, _ oci.Client, c *containers.Container, s *specs.Spec) error {
+			if len(devices) == 0 {
+				return nil
+			}
+			if err := cdi.Refresh(); err != nil {
+				log.G(ctx).Warnf("CDI registry refresh failed: %v", err)
+			}
+			bklog.G(ctx).Debugf("Injecting CDI devices %v", devices)
+			if _, err := cdi.InjectDevices(s, devices...); err != nil {
+				return errors.Wrapf(err, "CDI device injection failed")
+			}
+			// One crucial thing to keep in mind is that CDI device injection
+			// might add OCI Spec environment variables, hooks, and mounts as
+			// well. Therefore, it is important that none of the corresponding
+			// OCI Spec fields are reset up in the call stack once we return.
+			return nil
+		}
+	}
+
+	return []oci.SpecOpts{
+		withStaticCDIRegistry(),
+		withCDIDevices(dd...),
+	}, nil
+}
+
 // withDefaultProfile sets the default seccomp profile to the spec.
 // Note: must follow the setting of process capabilities
 func withDefaultProfile() oci.SpecOpts {

executor/oci/spec_windows.go

@@ -110,3 +110,11 @@ func sub(m mount.Mount, subPath string) (mount.Mount, func() error, error) {
 	m.Source = src
 	return m, func() error { return nil }, nil
 }
+
+func generateCDIOpts(ctx context.Context, devices []*pb.CDIDevice) ([]oci.SpecOpts, error) {
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	// https://github.com/cncf-tags/container-device-interface/issues/28
+	return nil, errors.New("no support for CDI on Windows")
+}