diff --git a/README.md b/README.md index 6dd34f3b4..bf2d3f9ea 100644 --- a/README.md +++ b/README.md @@ -233,14 +233,14 @@ The summary of resources available via plugins in this repository is given in th **Device Namespace : Registered Resource(s)** * `dlb.intel.com` : `pf` or `vf` * [dlb-libdlb-demo-pod.yaml](demo/dlb-libdlb-demo-pod.yaml) - * `dsa.intel.com` : `wq-user-[shared or dedicated]` + * `dsa.intel.com` : `wq-user-[shared|dedicated]` or `vfio` * [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml) * [dsa-dpdk-dmadevtest.yaml](demo/dsa-dpdk-dmadevtest.yaml) * `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings) * [intelfpga-job.yaml](demo/intelfpga-job.yaml) * `gpu.intel.com` : `i915`, `i915_monitoring`, `xe` or `xe_monitoring` * [intelgpu-job.yaml](demo/intelgpu-job.yaml) - * `iaa.intel.com` : `wq-user-[shared or dedicated]` + * `iaa.intel.com` : `wq-user-[shared|dedicated]` * [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml) * `npu.intel.com` : `accel` * [intel-npu-workload.yaml](demo/intel-npu-workload.yaml) diff --git a/cmd/dsa_plugin/README.md b/cmd/dsa_plugin/README.md index 5fe745333..1ee866d82 100644 --- a/cmd/dsa_plugin/README.md +++ b/cmd/dsa_plugin/README.md @@ -60,6 +60,20 @@ To create a custom provisioning config: $ kubectl create configmap --namespace=inteldeviceplugins-system intel-dsa-config --from-file=demo/dsa.conf ``` +#### VFIO Support + +Instead of using the default `idxd` driver based device resources, some workloads (e.g., DPDK) support using DSA through `vfio-pci` too. The DSA device plugin looks for VFIO +device resources when started with `-driver vfio-pci` parameter. The registered resources are `dsa.intel.com/vfio`. + +The sample idxd initcontainer can be used to bind the devices to `vfio-pci` with: + +```bash +$ kubectl apply -k deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/ +``` + +> **Note:**: The `vfio-pci` module must be loaded with `disable_denylist=1` parameter +> for the DSA device plugin to work correctly with DSA devices with `PCI ID=0b25`. + ### Verify Plugin Registration You can verify the plugin has been registered with the expected nodes by searching for the relevant resource allocation status on the nodes: diff --git a/cmd/dsa_plugin/dsa_plugin.go b/cmd/dsa_plugin/dsa_plugin.go index 53febbae0..d818b5428 100644 --- a/cmd/dsa_plugin/dsa_plugin.go +++ b/cmd/dsa_plugin/dsa_plugin.go @@ -1,4 +1,4 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. +// Copyright 2020-2026 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/idxd" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/vfio" "k8s.io/klog/v2" ) @@ -31,12 +32,18 @@ const ( devDir = "/dev/dsa" // Glob pattern for the state sysfs entry. statePattern = "/sys/bus/dsa/devices/dsa*/wq*/state" + + pciDevicesDir = "/sys/bus/pci/devices" ) func main() { - var sharedDevNum int + var ( + sharedDevNum int + plugin dpapi.Scanner + ) flag.IntVar(&sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same work queue") + dsaDriver := flag.String("driver", "idxd", "Device driver used for the DSA devices") flag.Parse() if sharedDevNum < 1 { @@ -44,9 +51,19 @@ func main() { os.Exit(1) } - plugin := idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum) - if plugin == nil { - klog.Fatal("Cannot create device plugin, please check above error messages.") + switch *dsaDriver { + case "idxd": + plugin = idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum) + case "vfio-pci": + dsaDeviceIDs := vfio.DeviceIDSet{ + "0x0b25": {}, + "0x11fb": {}, + "0x1212": {}, + } + plugin = vfio.NewDevicePlugin(pciDevicesDir, dsaDeviceIDs) + default: + klog.Warningf("Unsupported DSA driver: %s. Use either idxd or vfio-pci.", *dsaDriver) + os.Exit(1) } manager := dpapi.NewManager(namespace, plugin) diff --git a/demo/dsa-dpdk-dmadevtest-vfio.yaml b/demo/dsa-dpdk-dmadevtest-vfio.yaml new file mode 100644 index 000000000..a05c249c2 --- /dev/null +++ b/demo/dsa-dpdk-dmadevtest-vfio.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dpdk-vfio +spec: + restartPolicy: Never + containers: + - name: dpdk-vfio + image: intel/dsa-dpdk-dmadevtest:devel + command: + - dpdk-test + args: + - -a + - $(VFIO_BDF0) + - dmadev_autotest + securityContext: + capabilities: + add: ["IPC_LOCK"] + drop: ["ALL"] + volumeMounts: + - mountPath: /mnt/hugepage + name: hugepage + resources: + requests: + hugepages-2Mi: 64Mi + memory: 128Mi + dsa.intel.com/vfio: 1 + limits: + hugepages-2Mi: 64Mi + memory: 128Mi + dsa.intel.com/vfio: 1 + volumes: + - name: hugepage + emptyDir: + medium: HugePages diff --git a/demo/idxd-init.sh b/demo/idxd-init.sh index 0ef324d79..320ba64a1 100755 --- a/demo/idxd-init.sh +++ b/demo/idxd-init.sh @@ -3,7 +3,9 @@ set -euo pipefail DEV="${DEVICE_TYPE:-dsa}" +DSA_DRIVER=${DSA_DRIVER:-idxd} NODE_NAME="${NODE_NAME:-}" +DSA_PCI_IDS=${DSA_PCI_IDS:-0b25 11fb 1212} OPT="" [ "$DEV" != "dsa" ] && OPT="-v" @@ -14,6 +16,40 @@ function cmd() { "${@}" } +function bind_driver() { + NEW_DRIVER=$1 + DSA_PFS=$2 + + DEVS="" + for DEV in $(realpath /sys/bus/pci/devices/*); do + for PF in $DSA_PFS; do + if grep -q "$PF" "$DEV"/device; then + DEVS="$DEV $DEVS" + fi + done + done + + for D in $DEVS; do + BSF=$(basename "$D") + if [ -e "$D/driver" ]; then + P=$(realpath -L "$D/driver") + DRIVER=$(basename "$P") + else + DRIVER="" + fi + + if [ "$DRIVER" != "$NEW_DRIVER" ]; then + if [ -n "$DRIVER" ]; then + echo -n "$BSF" >/sys/bus/pci/drivers/"$DRIVER"/unbind + fi + echo -n "$NEW_DRIVER" >/sys/bus/pci/devices/"$BSF"/driver_override + echo -n "$BSF" >/sys/bus/pci/drivers/"$NEW_DRIVER"/bind + fi + done +} + +[[ "$DEV" == "dsa" && -e /sys/bus/pci/drivers/"$DSA_DRIVER" ]] && bind_driver "$DSA_DRIVER" "$DSA_PCI_IDS" + for i in $(accel-config list | jq -r '.[].dev' | grep ${OPT} "dsa"); do cmd accel-config disable-device "$i" diff --git a/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/devfs_path.yaml b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/devfs_path.yaml new file mode 100644 index 000000000..a9ecb96fa --- /dev/null +++ b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/devfs_path.yaml @@ -0,0 +1,11 @@ +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + name: devfs-vfio + mountPath: /dev/vfio +- op: add + path: /spec/template/spec/volumes/- + value: + name: devfs-vfio + hostPath: + path: /dev/vfio diff --git a/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/driver_args.yaml b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/driver_args.yaml new file mode 100644 index 000000000..c669673e6 --- /dev/null +++ b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/driver_args.yaml @@ -0,0 +1,4 @@ +- op: add + path: /spec/template/spec/containers/0/args + value: + - "-driver=vfio-pci" diff --git a/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/dsa_initcontainer.yaml b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/dsa_initcontainer.yaml new file mode 100644 index 000000000..0ec064e9a --- /dev/null +++ b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/dsa_initcontainer.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-dsa-plugin +spec: + template: + spec: + initContainers: + - name: intel-idxd-config-initcontainer + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DSA_DRIVER + value: vfio-pci + image: intel/intel-idxd-config-initcontainer:devel + securityContext: + seLinuxOptions: + type: "container_device_plugin_init_t" + readOnlyRootFilesystem: true + privileged: true + volumeMounts: + - mountPath: /sys/bus/dsa + name: sys-bus-dsa + - mountPath: /sys/devices + name: sys-devices + - mountPath: /idxd-init/scratch + name: scratch + volumes: + - name: sys-bus-dsa + hostPath: + path: /sys/bus/dsa + - name: sys-devices + hostPath: + path: /sys/devices + - name: scratch + emptyDir: {} diff --git a/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/kustomization.yaml b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/kustomization.yaml new file mode 100644 index 000000000..76d6cc76b --- /dev/null +++ b/deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/kustomization.yaml @@ -0,0 +1,12 @@ +resources: +- ../../base +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: dsa_initcontainer.yaml +- path: driver_args.yaml + target: + kind: DaemonSet +- path: devfs_path.yaml + target: + kind: DaemonSet diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml index 43219cdae..558681991 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml @@ -55,6 +55,12 @@ spec: spec: description: DsaDevicePluginSpec defines the desired state of DsaDevicePlugin. properties: + driver: + description: Driver name used for the DSA devices. + enum: + - idxd + - vfio-pci + type: string image: description: Image is a container image with DSA device plugin executable. type: string diff --git a/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go b/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go index 50a6eea5e..15865ceaf 100644 --- a/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go @@ -37,6 +37,10 @@ type DsaDevicePluginSpec struct { // ProvisioningConfig is a ConfigMap used to pass the DSA devices and workqueues configuration into idxd-config initcontainer. ProvisioningConfig string `json:"provisioningConfig,omitempty"` + // Driver name used for the DSA devices. + // +kubebuilder:validation:Enum=idxd;vfio-pci + Driver string `json:"driver,omitempty"` + // Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint. Tolerations []v1.Toleration `json:"tolerations,omitempty"` diff --git a/pkg/controllers/dsa/controller.go b/pkg/controllers/dsa/controller.go index c4f4e294b..adb88d51f 100644 --- a/pkg/controllers/dsa/controller.go +++ b/pkg/controllers/dsa/controller.go @@ -38,6 +38,8 @@ const ( ownerKey = ".metadata.controller.dsa" initcontainerName = "intel-idxd-config-initcontainer" configVolumeName = "intel-dsa-config-volume" + vfioDriver = "vfio-pci" + devfsVolumeName = "devfs" ) var defaultNodeSelector = deployments.DSAPluginDaemonSet().Spec.Template.Spec.NodeSelector @@ -101,6 +103,11 @@ func removeInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) func addInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) { yes := true + driver := "idxd" + + if dp.Spec.Driver == vfioDriver { + driver = vfioDriver + } ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, v1.Container{ Image: dp.Spec.InitImage, @@ -119,6 +126,10 @@ func addInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) { Name: "DEVICE_TYPE", Value: "dsa", }, + { + Name: "DSA_DRIVER", + Value: driver, + }, }, SecurityContext: &v1.SecurityContext{ SELinuxOptions: &v1.SELinuxOptions{ @@ -207,6 +218,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { addInitContainer(daemonSet, devicePlugin) } + devfsPath := "/dev/dsa" + if devicePlugin.Spec.Driver == vfioDriver { + devfsPath = "/dev/vfio" + } + + for _, volume := range daemonSet.Spec.Template.Spec.Volumes { + if volume.Name == devfsVolumeName { + volume.VolumeSource.HostPath.Path = devfsPath + break + } + } + + for _, volumePath := range daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts { + if volumePath.Name == devfsVolumeName { + volumePath.MountPath = devfsPath + break + } + } + if len(c.args.ImagePullSecretName) > 0 { daemonSet.Spec.Template.Spec.ImagePullSecrets = []v1.LocalObjectReference{ {Name: c.args.ImagePullSecretName}, @@ -285,6 +315,25 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( updated = true } + devfsPath := "/dev/dsa" + if dp.Spec.Driver == vfioDriver { + devfsPath = "/dev/vfio" + } + + for _, volume := range ds.Spec.Template.Spec.Volumes { + if volume.Name == devfsVolumeName { + volume.VolumeSource.HostPath.Path = devfsPath + break + } + } + + for _, volumePath := range ds.Spec.Template.Spec.Containers[0].VolumeMounts { + if volumePath.Name == devfsVolumeName { + volumePath.MountPath = devfsPath + break + } + } + return updated } @@ -320,9 +369,15 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node } func getPodArgs(gdp *devicepluginv1.DsaDevicePlugin) []string { - args := make([]string, 0, 4) + args := make([]string, 0, 6) args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel)) + if gdp.Spec.Driver == vfioDriver { + args = append(args, "-driver", "vfio-pci") + } else { + args = append(args, "-driver", "idxd") + } + if gdp.Spec.SharedDevNum > 0 { args = append(args, "-shared-dev-num", strconv.Itoa(gdp.Spec.SharedDevNum)) } else { diff --git a/pkg/vfio/plugin.go b/pkg/vfio/plugin.go new file mode 100644 index 000000000..67acfe5a2 --- /dev/null +++ b/pkg/vfio/plugin.go @@ -0,0 +1,182 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vfio + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + "github.com/pkg/errors" + "k8s.io/klog/v2" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +const ( + // VFIO devices directory and control device path. + vfioPath = "/dev/vfio" + vfioCtrlPath = "/dev/vfio/vfio" + // Frequency of device scans. + scanFrequency = 15 * time.Second + envVarPrefix = "VFIO_BDF" +) + +// DevicePlugin defines properties of the vfio device plugin. +type DevicePlugin struct { + scanTicker *time.Ticker + scanDone chan bool + devIDs DeviceIDSet + devDir string +} + +type DeviceIDSet map[string]struct{} + +// NewDevicePlugin creates DevicePlugin. +func NewDevicePlugin(devDir string, devIDs DeviceIDSet) *DevicePlugin { + return &DevicePlugin{ + devDir: devDir, + devIDs: devIDs, + scanTicker: time.NewTicker(scanFrequency), + scanDone: make(chan bool, 1), + } +} + +// Scan discovers devices and reports them to the upper level API. +func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error { + defer dp.scanTicker.Stop() + + for { + devTree, err := dp.scan() + if err != nil { + return err + } + + notifier.Notify(devTree) + + select { + case <-dp.scanDone: + return nil + case <-dp.scanTicker.C: + } + } +} + +func readFile(fpath string) (string, error) { + data, err := os.ReadFile(fpath) + if err != nil { + return "", errors.WithStack(err) + } + + return strings.TrimSpace(string(data)), nil +} + +// PostAllocate implements PostAllocator interface for vfio device plugin. It re-maps +// VFIO_BDF environment variables set by scan() to VFIO_BDF<0,1, ...> +// based on device resources requested by the container. +func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error { + tempMap := make(map[string]string) + + for _, cresp := range response.ContainerResponses { + counter := 0 + + for k := range cresp.Envs { + tempMap[strings.Join([]string{envVarPrefix, strconv.Itoa(counter)}, "")] = cresp.Envs[k] + counter++ + } + + cresp.Envs = tempMap + } + + return nil +} + +// scan collects devices by scanning sysfs and devfs entries. +func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { + // scan sysfs tree + pciDevices, err := filepath.Glob(filepath.Join(dp.devDir, "????:??:??.?")) + if err != nil { + return nil, errors.WithStack(err) + } + + devTree := dpapi.NewDeviceTree() + devNum := 0 + + for _, dpath := range pciDevices { + devID, err := readFile(filepath.Join(dpath, "device")) + if err != nil { + return nil, err + } + + if _, ok := dp.devIDs[devID]; !ok { + continue + } + + // device belongs to an IOMMU group + iommu_group, err := filepath.EvalSymlinks(filepath.Join(dpath, "iommu_group")) + if err != nil { + return nil, errors.WithStack(err) + } + + driver, err := filepath.EvalSymlinks(filepath.Join(dpath, "driver")) + if err != nil { + return nil, errors.WithStack(err) + } + + if filepath.Base(driver) != "vfio-pci" { + continue + } + + devNodes := []pluginapi.DeviceSpec{ + { + HostPath: filepath.Join(vfioPath, filepath.Base(iommu_group)), + ContainerPath: filepath.Join(vfioPath, filepath.Base(iommu_group)), + Permissions: "rw", + }, + { + HostPath: vfioCtrlPath, + ContainerPath: vfioCtrlPath, + Permissions: "rw", + }, + } + + // TODO: add IOMMUFD nodes + // iommuFdDevices, err := filepath.Glob(filepath.Join(dpath, "vfio-dev", "vfio?")) + // if err == nil { + // for _, iommuDev := range iommuFdDevices { + // devNodes = append(devNodes, pluginapi.DeviceSpec{ + // HostPath: filepath.Join(vfioPath, "devices", filepath.Base(iommuDev)), + // ContainerPath: filepath.Join(vfioPath, "devices", filepath.Base(iommuDev)), + // Permissions: "rw", + // }) + // } + // } + + devNum = devNum + 1 + bdf := filepath.Base(dpath) + + envs := map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, devNum): bdf, + } + + klog.V(4).Infof("%s (ID=%s): nodes: %+v", bdf, devID, devNodes) + devTree.AddDevice("vfio", bdf, dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, nil, envs, nil, nil)) + } + + return devTree, nil +} diff --git a/pkg/vfio/plugin_test.go b/pkg/vfio/plugin_test.go new file mode 100644 index 000000000..73e688e89 --- /dev/null +++ b/pkg/vfio/plugin_test.go @@ -0,0 +1,285 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vfio + +import ( + "flag" + "os" + "path" + "slices" + "testing" + + "github.com/pkg/errors" + + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" +) + +func init() { + _ = flag.Set("v", "4") //Enable debug output +} + +func createTestFiles(prefix string, dirs []string, files map[string][]byte, symlinks map[string]string) error { + for _, dir := range dirs { + err := os.MkdirAll(path.Join(prefix, dir), 0750) + if err != nil { + return errors.Wrap(err, "Failed to create fake device directory") + } + } + + for filename, body := range files { + err := os.WriteFile(path.Join(prefix, filename), body, 0600) + if err != nil { + return errors.Wrap(err, "Failed to create fake vendor file") + } + } + + for link, target := range symlinks { + err := os.MkdirAll(path.Join(prefix, target), 0750) + if err != nil { + return errors.Wrap(err, "Failed to create fake symlink target directory") + } + + err = os.Symlink(path.Join(prefix, target), path.Join(prefix, link)) + if err != nil { + return errors.Wrap(err, "Failed to create fake symlink") + } + } + + return nil +} + +// fakeNotifier implements Notifier interface. +type fakeNotifier struct { + scanDone chan bool + tree dpapi.DeviceTree +} + +// Notify stops plugin Scan. +func (n *fakeNotifier) Notify(newDeviceTree dpapi.DeviceTree) { + n.tree = newDeviceTree + n.scanDone <- true +} + +func TestScan(t *testing.T) { + tcases := []struct { + name string + deviceIDSet DeviceIDSet + files map[string][]byte + symlinks map[string]string + dirs []string + expectedDevNum int + expectedErr bool + }{ + { + name: "No error returned for uninitialized device plugin", + }, + { + name: "PCI Device ID reading fails", + deviceIDSet: DeviceIDSet{"0x37c8": {}}, + dirs: []string{ + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/nodevice": []byte("0x37c8"), + }, + expectedErr: true, + }, + { + name: "One PCI Device bound to vfio-pci and the Device ID matches", + deviceIDSet: DeviceIDSet{"0x37c8": {}}, + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c8"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:00.0/driver": "sys/bus/pci/drivers/vfio-pci", + }, + expectedDevNum: 1, + }, + { + name: "PCI Device ID matches but the device is not bound to vfio-pci", + deviceIDSet: DeviceIDSet{"0x1212": {}}, + dirs: []string{ + "sys/bus/pci/drivers/idxd", + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x1212"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:00.0/driver": "sys/bus/pci/drivers/idxd", + }, + }, + { + name: "PCI Device ID matches but the device is not bound to any driver", + deviceIDSet: DeviceIDSet{"0x1212": {}}, + dirs: []string{ + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x1212"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + }, + expectedErr: true, + }, + { + name: "One PCI Device bound to vfio-pci but the Device ID does not match", + deviceIDSet: DeviceIDSet{"0x37c8": {}}, + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x4940"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:00.0/driver": "sys/bus/pci/drivers/vfio-pci", + }, + }, + { + name: "PCI Device does not belong to any IOMMU group", + deviceIDSet: DeviceIDSet{"0x37c8": {}}, + dirs: []string{ + "sys/bus/pci/devices/0000:02:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c8"), + }, + expectedErr: true, + }, + { + name: "Two PCI Devices bound to vfio-pci but only one Device ID matches", + deviceIDSet: DeviceIDSet{"0x37c8": {}}, + dirs: []string{ + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:03:00.0", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c8"), + "sys/bus/pci/devices/0000:03:00.0/device": []byte("0x4940"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:00.0/driver": "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:03:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:03:00.0/driver": "sys/bus/pci/drivers/vfio-pci", + }, + expectedDevNum: 1, + }, + } + + for _, tt := range tcases { + t.Run(tt.name, func(t *testing.T) { + tmpdir, err := os.MkdirTemp("/tmp/", "vfioplugin-TestScanPrivate-*") + if err != nil { + t.Fatal(err) + } + + if err = createTestFiles(tmpdir, tt.dirs, tt.files, tt.symlinks); err != nil { + t.Fatalf("%+v", err) + } + + dp := NewDevicePlugin( + path.Join(tmpdir, "sys/bus/pci/devices"), + tt.deviceIDSet, + ) + + fN := fakeNotifier{ + scanDone: dp.scanDone, + } + + err = dp.Scan(&fN) + + if tt.expectedErr && err == nil { + t.Errorf("expected error, but got success") + } + if !tt.expectedErr && err != nil { + t.Errorf("got unexpected error: %+v", err) + } + devNum := 0 + for _, resource := range fN.tree { + devNum = devNum + len(resource) + } + if devNum != tt.expectedDevNum { + t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, devNum) + } + + if err = os.RemoveAll(tmpdir); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestPostAllocate(t *testing.T) { + response := new(pluginapi.AllocateResponse) + cresp := new(pluginapi.ContainerAllocateResponse) + response.ContainerResponses = append(response.ContainerResponses, cresp) + testMap := map[string]string{ + "VFIO_BDF29": "03:04.1", + "VFIO_BDF13": "03:04.2", + "VFIO_BDF6": "03:04.3", + "VFIO_BDF21": "03:04.4", + } + response.ContainerResponses[0].Envs = testMap + resultKey := []string{ + "VFIO_BDF0", + "VFIO_BDF1", + "VFIO_BDF2", + "VFIO_BDF3", + } + expectedValues := map[string]struct{}{ + "03:04.1": {}, + "03:04.2": {}, + "03:04.3": {}, + "03:04.4": {}, + } + + dp := &DevicePlugin{} + if err := dp.PostAllocate(response); err != nil { + t.Errorf("Unexpected error: %+v", err) + } + + if len(response.ContainerResponses[0].Envs) != 4 { + t.Fatal("Set wrong number of Environment Variables") + } + + for k := range response.ContainerResponses[0].Envs { + if !slices.Contains(resultKey, k) { + t.Fatalf("Set wrong key: %s. The key should be in the range %v", k, resultKey) + } + } + + for _, key := range resultKey { + if value, ok := response.ContainerResponses[0].Envs[key]; ok { + if _, ok := expectedValues[value]; ok { + delete(expectedValues, value) + } else { + t.Errorf("Unexpected value %s", value) + } + } + } +} diff --git a/test/e2e/dsa/dsa.go b/test/e2e/dsa/dsa.go index c5eaf23ec..452dda20e 100644 --- a/test/e2e/dsa/dsa.go +++ b/test/e2e/dsa/dsa.go @@ -1,4 +1,4 @@ -// Copyright 2021-2022 Intel Corporation. All Rights Reserved. +// Copyright 2021-2026 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import ( "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/test/e2e/framework" e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" @@ -34,11 +35,14 @@ const ( ns = "inteldeviceplugins-system" timeout = time.Second * 120 kustomizationYaml = "deployments/dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml" + kustomVfioYaml = "deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/dsa_initcontainer.yaml" configmapYaml = "demo/dsa.conf" demoYaml = "demo/dsa-accel-config-demo-pod.yaml" dpdkDemoYaml = "demo/dsa-dpdk-dmadevtest.yaml" + dpdkVfioYaml = "demo/dsa-dpdk-dmadevtest-vfio.yaml" podName = "dsa-accel-config-demo" dpdkPodName = "dpdk" + dpdkVfioPodName = "dpdk-vfio" ) func init() { @@ -49,15 +53,11 @@ func describe() { f := framework.NewDefaultFramework("dsaplugin") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - kustomizationPath, errFailedToLocateRepoFile := utils.LocateRepoFile(kustomizationYaml) - if errFailedToLocateRepoFile != nil { - framework.Failf("unable to locate %q: %v", kustomizationYaml, errFailedToLocateRepoFile) - } - - configmap, errFailedToLocateRepoFile := utils.LocateRepoFile(configmapYaml) - if errFailedToLocateRepoFile != nil { - framework.Failf("unable to locate %q: %v", configmapYaml, errFailedToLocateRepoFile) - } + var errFailedToLocateRepoFile error + var dpPodName string + var kustomizationPath string + var configMapPath string + var expectedResource corev1.ResourceName demoPath, errFailedToLocateRepoFile := utils.LocateRepoFile(demoYaml) if errFailedToLocateRepoFile != nil { @@ -69,46 +69,59 @@ func describe() { framework.Failf("unable to locate %q: %v", dpdkDemoYaml, errFailedToLocateRepoFile) } - var dpPodName string - - ginkgo.BeforeEach(func(ctx context.Context) { - ginkgo.By("deploying DSA plugin") - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configmap) - - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath)) - - ginkgo.By("waiting for DSA plugin's availability") - podList, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, f.Namespace.Name, - labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second) - if err != nil { - e2edebug.DumpAllNamespaceInfo(ctx, f.ClientSet, f.Namespace.Name) - e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf) - framework.Failf("unable to wait for all pods to be running and ready: %v", err) - } - dpPodName = podList.Items[0].Name - - ginkgo.By("checking DSA plugin's securityContext") - if err = utils.TestPodsFileSystemInfo(podList.Items); err != nil { - framework.Failf("container filesystem info checks failed: %v", err) - } - }) - - ginkgo.AfterEach(func(ctx context.Context) { - ginkgo.By("undeploying DSA plugin") - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "-k", filepath.Dir(kustomizationPath)) - if err := e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, dpPodName, f.Namespace.Name, 30*time.Second); err != nil { - framework.Failf("failed to terminate pod: %v", err) - } - }) + demoDpdkVfioPath, errFailedToLocateRepoFile := utils.LocateRepoFile(dpdkVfioYaml) + if errFailedToLocateRepoFile != nil { + framework.Failf("unable to locate %q: %v", dpdkDemoYaml, errFailedToLocateRepoFile) + } ginkgo.Context("When DSA resources are available [Resource:dedicated]", func() { ginkgo.BeforeEach(func(ctx context.Context) { + kustomizationPath, errFailedToLocateRepoFile = utils.LocateRepoFile(kustomizationYaml) + if errFailedToLocateRepoFile != nil { + framework.Failf("unable to locate %q: %v", kustomizationYaml, errFailedToLocateRepoFile) + } + + configMapPath, errFailedToLocateRepoFile = utils.LocateRepoFile(configmapYaml) + if errFailedToLocateRepoFile != nil { + framework.Failf("unable to locate %q: %v", configmapYaml, errFailedToLocateRepoFile) + } + + expectedResource = "dsa.intel.com/wq-user-dedicated" + + ginkgo.By("deploying DSA plugin") + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configMapPath) + + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath)) + + ginkgo.By("waiting for DSA plugin's availability") + podList, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, f.Namespace.Name, + labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second) + if err != nil { + e2edebug.DumpAllNamespaceInfo(ctx, f.ClientSet, f.Namespace.Name) + e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf) + framework.Failf("unable to wait for all pods to be running and ready: %v", err) + } + dpPodName = podList.Items[0].Name + + ginkgo.By("checking DSA plugin's securityContext") + if err = utils.TestPodsFileSystemInfo(podList.Items); err != nil { + framework.Failf("container filesystem info checks failed: %v", err) + } + ginkgo.By("checking if the resource is allocatable") - if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second, utils.WaitForPositiveResource); err != nil { + if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, expectedResource, 300*time.Second, utils.WaitForPositiveResource); err != nil { framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err) } }) + ginkgo.AfterEach(func(ctx context.Context) { + ginkgo.By("undeploying DSA plugin and its ConfigMap") + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "-k", filepath.Dir(kustomizationPath)) + if err := e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, dpPodName, f.Namespace.Name, 30*time.Second); err != nil { + framework.Failf("failed to terminate pod: %v", err) + } + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "configmap", "intel-dsa-config") + }) ginkgo.It("deploys a demo app [App:accel-config]", func(ctx context.Context) { e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoPath) @@ -129,4 +142,59 @@ func describe() { ginkgo.It("does nothing", func() {}) }) }) + + ginkgo.Context("When DSA VFIO resources are available [Resource:vfio]", func() { + ginkgo.BeforeEach(func(ctx context.Context) { + kustomizationPath, errFailedToLocateRepoFile = utils.LocateRepoFile(kustomVfioYaml) + if errFailedToLocateRepoFile != nil { + framework.Failf("unable to locate %q: %v", kustomVfioYaml, errFailedToLocateRepoFile) + } + + expectedResource = "dsa.intel.com/vfio" + + ginkgo.By("deploying DSA plugin") + + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath)) + + ginkgo.By("waiting for DSA plugin's availability") + podList, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, f.Namespace.Name, + labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second) + if err != nil { + e2edebug.DumpAllNamespaceInfo(ctx, f.ClientSet, f.Namespace.Name) + e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf) + framework.Failf("unable to wait for all pods to be running and ready: %v", err) + } + dpPodName = podList.Items[0].Name + + ginkgo.By("checking DSA plugin's securityContext") + if err = utils.TestPodsFileSystemInfo(podList.Items); err != nil { + framework.Failf("container filesystem info checks failed: %v", err) + } + + ginkgo.By("checking if the resource is allocatable") + if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, expectedResource, 300*time.Second, utils.WaitForPositiveResource); err != nil { + framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err) + } + }) + + ginkgo.AfterEach(func(ctx context.Context) { + ginkgo.By("undeploying DSA plugin and its ConfigMap") + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "delete", "-k", filepath.Dir(kustomizationPath)) + if err := e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, dpPodName, f.Namespace.Name, 30*time.Second); err != nil { + framework.Failf("failed to terminate pod: %v", err) + } + }) + + ginkgo.It("deploys a demo app [App:dpdk-vfio-test]", func(ctx context.Context) { + e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoDpdkVfioPath) + + ginkgo.By("waiting for the DSA DPDK VFIO demo to succeed") + err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, dpdkVfioPodName, f.Namespace.Name, 200*time.Second) + gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, dpdkVfioPodName, dpdkVfioPodName)) + }) + + ginkgo.When("there is no app to run [App:noapp]", func() { + ginkgo.It("does nothing", func() {}) + }) + }) } diff --git a/test/envtest/dsadeviceplugin_controller_test.go b/test/envtest/dsadeviceplugin_controller_test.go index c37ed8c24..9aa150f3c 100644 --- a/test/envtest/dsadeviceplugin_controller_test.go +++ b/test/envtest/dsadeviceplugin_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2020-2022 Intel Corporation. All Rights Reserved. +// Copyright 2020-2026 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -81,6 +81,7 @@ var _ = Describe("DsaDevicePlugin Controller", func() { updatedProvisioningConfig := "updated-dsa-provisioningconfig" updatedLogLevel := 2 updatedSharedDevNum := 42 + updatedDriver := "vfio-pci" updatedNodeSelector := map[string]string{"updated-dsa-nodeselector": "true"} fetched.Spec.Image = updatedImage @@ -88,6 +89,7 @@ var _ = Describe("DsaDevicePlugin Controller", func() { fetched.Spec.ProvisioningConfig = updatedProvisioningConfig fetched.Spec.LogLevel = updatedLogLevel fetched.Spec.SharedDevNum = updatedSharedDevNum + fetched.Spec.Driver = updatedDriver fetched.Spec.NodeSelector = updatedNodeSelector Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) @@ -106,6 +108,8 @@ var _ = Describe("DsaDevicePlugin Controller", func() { expectArgs := []string{ "-v", strconv.Itoa(updatedLogLevel), + "-driver", + "vfio-pci", "-shared-dev-num", strconv.Itoa(updatedSharedDevNum), } @@ -123,6 +127,8 @@ var _ = Describe("DsaDevicePlugin Controller", func() { Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(HaveLen(3)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env[2].Value).To(Equal("vfio-pci")) Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(expectedVolume)) Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))