add vfio based devices to DSA

README.md

@@ -233,14 +233,14 @@ The summary of resources available via plugins in this repository is given in th
 **Device Namespace : Registered Resource(s)**
  * `dlb.intel.com`  : `pf` or `vf`
    * [dlb-libdlb-demo-pod.yaml](demo/dlb-libdlb-demo-pod.yaml)
- * `dsa.intel.com`  : `wq-user-[shared or dedicated]`
+ * `dsa.intel.com`  : `wq-user-[shared|dedicated]` or `vfio`
    * [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml)
    * [dsa-dpdk-dmadevtest.yaml](demo/dsa-dpdk-dmadevtest.yaml)
  * `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)
    * [intelfpga-job.yaml](demo/intelfpga-job.yaml)
  * `gpu.intel.com`  : `i915`, `i915_monitoring`, `xe` or `xe_monitoring`
    * [intelgpu-job.yaml](demo/intelgpu-job.yaml)
- * `iaa.intel.com`  : `wq-user-[shared or dedicated]`
+ * `iaa.intel.com`  : `wq-user-[shared|dedicated]`
    * [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml)
  * `npu.intel.com`  : `accel`
    * [intel-npu-workload.yaml](demo/intel-npu-workload.yaml)

cmd/dsa_plugin/README.md

@@ -60,6 +60,20 @@ To create a custom provisioning config:
 $ kubectl create configmap --namespace=inteldeviceplugins-system intel-dsa-config --from-file=demo/dsa.conf
 ```
 
+#### VFIO Support
+
+Instead of using the default `idxd` driver based device resources, some workloads (e.g., DPDK) support using DSA through `vfio-pci` too. The DSA device plugin looks for VFIO
+device resources when started with `-driver vfio-pci` parameter. The registered resources are `dsa.intel.com/vfio`.
+
+The sample idxd initcontainer can be used to bind the devices to `vfio-pci` with:
+
+```bash
+$ kubectl apply -k deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/
+```
+
+> **Note:**: The `vfio-pci` module must be loaded with `disable_denylist=1` parameter
+> for the DSA device plugin to work correctly with DSA devices with `PCI ID=0b25`.
+
 ### Verify Plugin Registration
 You can verify the plugin has been registered with the expected nodes by searching for the relevant
 resource allocation status on the nodes:

cmd/dsa_plugin/dsa_plugin.go

@@ -1,4 +1,4 @@
-// Copyright 2020 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2026 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import (
 
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/idxd"
+	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/vfio"
 
 	"k8s.io/klog/v2"
 )
@@ -31,22 +32,38 @@ const (
 	devDir = "/dev/dsa"
 	// Glob pattern for the state sysfs entry.
 	statePattern = "/sys/bus/dsa/devices/dsa*/wq*/state"
+
+	pciDevicesDir = "/sys/bus/pci/devices"
 )
 
 func main() {
-	var sharedDevNum int
+	var (
+		sharedDevNum int
+		plugin       dpapi.Scanner
+	)
 
 	flag.IntVar(&sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same work queue")
+	dsaDriver := flag.String("driver", "idxd", "Device driver used for the DSA devices")
 	flag.Parse()
 
 	if sharedDevNum < 1 {
 		klog.Warning("The number of containers sharing the same work queue must be greater than zero")
 		os.Exit(1)
 	}
 
-	plugin := idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum)
-	if plugin == nil {
-		klog.Fatal("Cannot create device plugin, please check above error messages.")
+	switch *dsaDriver {
+	case "idxd":
+		plugin = idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum)
+	case "vfio-pci":
+		dsaDeviceIDs := vfio.DeviceIDSet{
+			"0x0b25": {},
+			"0x11fb": {},
+			"0x1212": {},
+		}
+		plugin = vfio.NewDevicePlugin(pciDevicesDir, dsaDeviceIDs)
+	default:
+		klog.Warningf("Unsupported DSA driver: %s. Use either idxd or vfio-pci.", *dsaDriver)
+		os.Exit(1)
 	}
 
 	manager := dpapi.NewManager(namespace, plugin)

demo/dsa-dpdk-dmadevtest-vfio.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-vfio
+spec:
+  restartPolicy: Never
+  containers:
+    - name: dpdk-vfio
+      image: intel/dsa-dpdk-dmadevtest:devel
+      command:
+        - dpdk-test
+      args:
+        - -a
+        - $(VFIO_BDF0)
+        - dmadev_autotest
+      securityContext:
+        capabilities:
+          add: ["IPC_LOCK"]
+          drop: ["ALL"]
+      volumeMounts:
+        - mountPath: /mnt/hugepage
+          name: hugepage
+      resources:
+        requests:
+          hugepages-2Mi: 64Mi
+          memory: 128Mi
+          dsa.intel.com/vfio: 1
+        limits:
+          hugepages-2Mi: 64Mi
+          memory: 128Mi
+          dsa.intel.com/vfio: 1
+  volumes:
+  - name: hugepage
+    emptyDir:
+      medium: HugePages

demo/idxd-init.sh

@@ -3,7 +3,9 @@
 set -euo pipefail
 
 DEV="${DEVICE_TYPE:-dsa}"
+DSA_DRIVER=${DSA_DRIVER:-idxd}
 NODE_NAME="${NODE_NAME:-}"
+DSA_PCI_IDS=${DSA_PCI_IDS:-0b25 11fb 1212}
 OPT=""
 [ "$DEV" != "dsa" ] && OPT="-v"
 
@@ -14,6 +16,40 @@ function cmd() {
     "${@}"
 }
 
+function bind_driver() {
+    NEW_DRIVER=$1
+    DSA_PFS=$2
+
+    DEVS=""
+    for DEV in $(realpath /sys/bus/pci/devices/*); do
+        for PF in $DSA_PFS; do
+            if grep -q "$PF" "$DEV"/device; then
+                DEVS="$DEV $DEVS"
+            fi
+        done
+    done
+
+    for D in $DEVS; do
+        BSF=$(basename "$D")
+        if [ -e "$D/driver" ]; then
+            P=$(realpath -L "$D/driver")
+            DRIVER=$(basename "$P")
+        else
+            DRIVER=""
+        fi
+
+        if [ "$DRIVER" != "$NEW_DRIVER" ]; then
+            if [ -n "$DRIVER" ]; then
+                echo -n "$BSF" >/sys/bus/pci/drivers/"$DRIVER"/unbind
+            fi
+            echo -n "$NEW_DRIVER" >/sys/bus/pci/devices/"$BSF"/driver_override
+            echo -n "$BSF" >/sys/bus/pci/drivers/"$NEW_DRIVER"/bind
+        fi
+    done
+}
+
+[[ "$DEV" == "dsa" && -e /sys/bus/pci/drivers/"$DSA_DRIVER" ]] && bind_driver "$DSA_DRIVER" "$DSA_PCI_IDS"
+
 for i in $(accel-config list | jq -r '.[].dev' | grep ${OPT} "dsa"); do
 
     cmd accel-config disable-device "$i"

deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/devfs_path.yaml

@@ -0,0 +1,11 @@
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: devfs-vfio
+    mountPath: /dev/vfio
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: devfs-vfio
+    hostPath:
+      path: /dev/vfio

deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/driver_args.yaml

@@ -0,0 +1,4 @@
+- op: add
+  path: /spec/template/spec/containers/0/args
+  value:
+    - "-driver=vfio-pci"

deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/dsa_initcontainer.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-dsa-plugin
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: intel-idxd-config-initcontainer
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: DSA_DRIVER
+            value: vfio-pci
+        image: intel/intel-idxd-config-initcontainer:devel
+        securityContext:
+          seLinuxOptions:
+            type: "container_device_plugin_init_t"
+          readOnlyRootFilesystem: true
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/bus/dsa
+          name: sys-bus-dsa
+        - mountPath: /sys/devices
+          name: sys-devices
+        - mountPath: /idxd-init/scratch
+          name: scratch
+      volumes:
+      - name: sys-bus-dsa
+        hostPath:
+          path: /sys/bus/dsa
+      - name: sys-devices
+        hostPath:
+          path: /sys/devices
+      - name: scratch
+        emptyDir: {}

deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/kustomization.yaml

@@ -0,0 +1,12 @@
+resources:
+- ../../base
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+patches:
+- path: dsa_initcontainer.yaml
+- path: driver_args.yaml
+  target:
+    kind: DaemonSet
+- path: devfs_path.yaml
+  target:
+    kind: DaemonSet

deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml

@@ -55,6 +55,12 @@ spec:
           spec:
             description: DsaDevicePluginSpec defines the desired state of DsaDevicePlugin.
             properties:
+              driver:
+                description: Driver name used for the DSA devices.
+                enum:
+                - idxd
+                - vfio-pci
+                type: string
               image:
                 description: Image is a container image with DSA device plugin executable.
                 type: string

pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go

@@ -37,6 +37,10 @@ type DsaDevicePluginSpec struct {
 	// ProvisioningConfig is a ConfigMap used to pass the DSA devices and workqueues configuration into idxd-config initcontainer.
 	ProvisioningConfig string `json:"provisioningConfig,omitempty"`
 
+	// Driver name used for the DSA devices.
+	// +kubebuilder:validation:Enum=idxd;vfio-pci
+	Driver string `json:"driver,omitempty"`
+
 	// Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint.
 	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
 

pkg/controllers/dsa/controller.go

@@ -38,6 +38,8 @@ const (
 	ownerKey          = ".metadata.controller.dsa"
 	initcontainerName = "intel-idxd-config-initcontainer"
 	configVolumeName  = "intel-dsa-config-volume"
+	vfioDriver        = "vfio-pci"
+	devfsVolumeName   = "devfs"
 )
 
 var defaultNodeSelector = deployments.DSAPluginDaemonSet().Spec.Template.Spec.NodeSelector
@@ -101,6 +103,11 @@ func removeInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin)
 
 func addInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) {
 	yes := true
+	driver := "idxd"
+
+	if dp.Spec.Driver == vfioDriver {
+		driver = vfioDriver
+	}
 
 	ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, v1.Container{
 		Image:           dp.Spec.InitImage,
@@ -119,6 +126,10 @@ func addInitContainer(ds *apps.DaemonSet, dp *devicepluginv1.DsaDevicePlugin) {
 				Name:  "DEVICE_TYPE",
 				Value: "dsa",
 			},
+			{
+				Name:  "DSA_DRIVER",
+				Value: driver,
+			},
 		},
 		SecurityContext: &v1.SecurityContext{
 			SELinuxOptions: &v1.SELinuxOptions{
@@ -207,6 +218,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 		addInitContainer(daemonSet, devicePlugin)
 	}
 
+	devfsPath := "/dev/dsa"
+	if devicePlugin.Spec.Driver == vfioDriver {
+		devfsPath = "/dev/vfio"
+	}
+
+	for _, volume := range daemonSet.Spec.Template.Spec.Volumes {
+		if volume.Name == devfsVolumeName {
+			volume.VolumeSource.HostPath.Path = devfsPath
+			break
+		}
+	}
+
+	for _, volumePath := range daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts {
+		if volumePath.Name == devfsVolumeName {
+			volumePath.MountPath = devfsPath
+			break
+		}
+	}
+
 	if len(c.args.ImagePullSecretName) > 0 {
 		daemonSet.Spec.Template.Spec.ImagePullSecrets = []v1.LocalObjectReference{
 			{Name: c.args.ImagePullSecretName},
@@ -285,6 +315,25 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	devfsPath := "/dev/dsa"
+	if dp.Spec.Driver == vfioDriver {
+		devfsPath = "/dev/vfio"
+	}
+
+	for _, volume := range ds.Spec.Template.Spec.Volumes {
+		if volume.Name == devfsVolumeName {
+			volume.VolumeSource.HostPath.Path = devfsPath
+			break
+		}
+	}
+
+	for _, volumePath := range ds.Spec.Template.Spec.Containers[0].VolumeMounts {
+		if volumePath.Name == devfsVolumeName {
+			volumePath.MountPath = devfsPath
+			break
+		}
+	}
+
 	return updated
 }
 
@@ -320,9 +369,15 @@ func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, node
 }
 
 func getPodArgs(gdp *devicepluginv1.DsaDevicePlugin) []string {
-	args := make([]string, 0, 4)
+	args := make([]string, 0, 6)
 	args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel))
 
+	if gdp.Spec.Driver == vfioDriver {
+		args = append(args, "-driver", "vfio-pci")
+	} else {
+		args = append(args, "-driver", "idxd")
+	}
+
 	if gdp.Spec.SharedDevNum > 0 {
 		args = append(args, "-shared-dev-num", strconv.Itoa(gdp.Spec.SharedDevNum))
 	} else {