Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -233,14 +233,14 @@ The summary of resources available via plugins in this repository is given in th
**Device Namespace : Registered Resource(s)**
* `dlb.intel.com` : `pf` or `vf`
* [dlb-libdlb-demo-pod.yaml](demo/dlb-libdlb-demo-pod.yaml)
* `dsa.intel.com` : `wq-user-[shared or dedicated]`
* `dsa.intel.com` : `wq-user-[shared|dedicated]` or `vfio`
* [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml)
* [dsa-dpdk-dmadevtest.yaml](demo/dsa-dpdk-dmadevtest.yaml)
* `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)
* [intelfpga-job.yaml](demo/intelfpga-job.yaml)
* `gpu.intel.com` : `i915`, `i915_monitoring`, `xe` or `xe_monitoring`
* [intelgpu-job.yaml](demo/intelgpu-job.yaml)
* `iaa.intel.com` : `wq-user-[shared or dedicated]`
* `iaa.intel.com` : `wq-user-[shared|dedicated]`
* [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml)
* `npu.intel.com` : `accel`
* [intel-npu-workload.yaml](demo/intel-npu-workload.yaml)
Expand Down
14 changes: 14 additions & 0 deletions cmd/dsa_plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ To create a custom provisioning config:
$ kubectl create configmap --namespace=inteldeviceplugins-system intel-dsa-config --from-file=demo/dsa.conf
```

#### VFIO Support

Instead of using the default `idxd` driver based device resources, some workloads (e.g., DPDK) support using DSA through `vfio-pci` too. The DSA device plugin looks for VFIO
device resources when started with `-driver vfio-pci` parameter. The registered resources are `dsa.intel.com/vfio`.

The sample idxd initcontainer can be used to bind the devices to `vfio-pci` with:

```bash
$ kubectl apply -k deployments/dsa_plugin/overlays/dsa_vfio_initcontainer/
```

> **Note:**: The `vfio-pci` module must be loaded with `disable_denylist=1` parameter
> for the DSA device plugin to work correctly with DSA devices with `PCI ID=0b25`.

### Verify Plugin Registration
You can verify the plugin has been registered with the expected nodes by searching for the relevant
resource allocation status on the nodes:
Expand Down
32 changes: 26 additions & 6 deletions cmd/dsa_plugin/dsa_plugin.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2020 Intel Corporation. All Rights Reserved.
// Copyright 2020-2026 Intel Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -20,6 +20,7 @@ import (

dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/idxd"
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/vfio"

"k8s.io/klog/v2"
)
Expand All @@ -31,22 +32,41 @@ const (
devDir = "/dev/dsa"
// Glob pattern for the state sysfs entry.
statePattern = "/sys/bus/dsa/devices/dsa*/wq*/state"

pciDevicesDir = "/sys/bus/pci/devices"
)

func main() {
var sharedDevNum int
var (
sharedDevNum int
plugin dpapi.Scanner
)

flag.IntVar(&sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same work queue")
dsaDriver := flag.String("driver", "idxd", "Device driver used for the DSA devices")
flag.Parse()

if sharedDevNum < 1 {
klog.Warning("The number of containers sharing the same work queue must be greater than zero")
klog.Warning("The number of containers sharing the same work queue must be greater than zero.")
os.Exit(1)
}

plugin := idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum)
if plugin == nil {
klog.Fatal("Cannot create device plugin, please check above error messages.")
switch *dsaDriver {
case "idxd":
plugin = idxd.NewDevicePlugin(statePattern, devDir, sharedDevNum)
case "vfio-pci":
dsaDeviceIDs := vfio.DeviceIDSet{
"0x0b25": {},
"0x11fb": {},
"0x1212": {},
}
if sharedDevNum > 1 {
klog.Warning("shared-dev-num setting ignored when using -driver=vfio-pci.")
}
plugin = vfio.NewDevicePlugin(pciDevicesDir, dsaDeviceIDs)
Comment thread
mythi marked this conversation as resolved.
default:
klog.Warningf("Unsupported DSA driver: %s. Use either idxd or vfio-pci.", *dsaDriver)
os.Exit(1)
}

manager := dpapi.NewManager(namespace, plugin)
Expand Down
35 changes: 35 additions & 0 deletions demo/dsa-dpdk-dmadevtest-vfio.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
apiVersion: v1
kind: Pod
metadata:
name: dpdk-vfio
spec:
restartPolicy: Never
containers:
- name: dpdk-vfio
image: intel/dsa-dpdk-dmadevtest:devel
command:
- dpdk-test
args:
- -a
- $(VFIO_BDF0)
- dmadev_autotest
securityContext:
capabilities:
add: ["IPC_LOCK"]
drop: ["ALL"]
volumeMounts:
- mountPath: /mnt/hugepage
name: hugepage
resources:
requests:
hugepages-2Mi: 64Mi
memory: 128Mi
dsa.intel.com/vfio: 1
limits:
hugepages-2Mi: 64Mi
memory: 128Mi
dsa.intel.com/vfio: 1
volumes:
- name: hugepage
emptyDir:
medium: HugePages
36 changes: 36 additions & 0 deletions demo/idxd-init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
set -euo pipefail

DEV="${DEVICE_TYPE:-dsa}"
DSA_DRIVER=${DSA_DRIVER:-idxd}
NODE_NAME="${NODE_NAME:-}"
DSA_PCI_IDS=${DSA_PCI_IDS:-0b25 11fb 1212}
Comment thread
mythi marked this conversation as resolved.
OPT=""
[ "$DEV" != "dsa" ] && OPT="-v"

Expand All @@ -14,6 +16,40 @@ function cmd() {
"${@}"
}

function bind_driver() {
NEW_DRIVER=$1
DSA_PFS=$2

DEVS=""
for PCIDEV in $(realpath /sys/bus/pci/devices/*); do
for PF in $DSA_PFS; do
if grep -q "$PF" "$PCIDEV"/device; then
DEVS="$PCIDEV $DEVS"
fi
done
done

for D in $DEVS; do
BSF=$(basename "$D")
if [ -e "$D/driver" ]; then
P=$(realpath -L "$D/driver")
DRIVER=$(basename "$P")
else
DRIVER=""
fi

if [ "$DRIVER" != "$NEW_DRIVER" ]; then
if [ -n "$DRIVER" ]; then
echo -n "$BSF" >/sys/bus/pci/drivers/"$DRIVER"/unbind
fi
echo -n "$NEW_DRIVER" >/sys/bus/pci/devices/"$BSF"/driver_override
echo -n "$BSF" >/sys/bus/pci/drivers/"$NEW_DRIVER"/bind
fi
done
}

[[ "$DEV" == "dsa" && -e /sys/bus/pci/drivers/"$DSA_DRIVER" ]] && bind_driver "$DSA_DRIVER" "$DSA_PCI_IDS"

for i in $(accel-config list | jq -r '.[].dev' | grep ${OPT} "dsa"); do

cmd accel-config disable-device "$i"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ spec:
volumeMounts:
- mountPath: /sys/bus/dsa
name: sys-bus-dsa
- mountPath: /sys/bus/pci
name: sys-bus-pci
- mountPath: /sys/devices
name: sys-devices
- mountPath: /idxd-init/conf
Expand All @@ -31,6 +33,9 @@ spec:
- name: sys-bus-dsa
hostPath:
path: /sys/bus/dsa
- name: sys-bus-pci
hostPath:
path: /sys/bus/pci
- name: sys-devices
hostPath:
path: /sys/devices
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
- op: add
path: /spec/template/spec/containers/0/volumeMounts/-
value:
name: devfs-vfio
mountPath: /dev/vfio
- op: add
path: /spec/template/spec/volumes/-
value:
name: devfs-vfio
hostPath:
path: /dev/vfio
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
- op: add
path: /spec/template/spec/containers/0/args
value:
- "-driver=vfio-pci"
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: intel-dsa-plugin
spec:
template:
spec:
initContainers:
- name: intel-idxd-config-initcontainer
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: DSA_DRIVER
value: vfio-pci
image: intel/intel-idxd-config-initcontainer:devel
securityContext:
seLinuxOptions:
type: "container_device_plugin_init_t"
readOnlyRootFilesystem: true
privileged: true
volumeMounts:
- mountPath: /sys/bus/dsa
name: sys-bus-dsa
- mountPath: /sys/bus/pci
name: sys-bus-pci
- mountPath: /sys/devices
name: sys-devices
- mountPath: /idxd-init/scratch
name: scratch
volumes:
- name: sys-bus-dsa
hostPath:
path: /sys/bus/dsa
- name: sys-bus-pci
hostPath:
path: /sys/bus/pci
- name: sys-devices
hostPath:
path: /sys/devices
- name: scratch
emptyDir: {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
resources:
- ../../base
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:
- path: dsa_initcontainer.yaml
- path: driver_args.yaml
target:
kind: DaemonSet
- path: devfs_path.yaml
target:
kind: DaemonSet
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ spec:
spec:
description: DsaDevicePluginSpec defines the desired state of DsaDevicePlugin.
properties:
driver:
default: idxd
description: Driver name used for the DSA devices.
enum:
- idxd
- vfio-pci
type: string
image:
description: Image is a container image with DSA device plugin executable.
type: string
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ type DsaDevicePluginSpec struct {
// ProvisioningConfig is a ConfigMap used to pass the DSA devices and workqueues configuration into idxd-config initcontainer.
ProvisioningConfig string `json:"provisioningConfig,omitempty"`

// Driver name used for the DSA devices.
// +kubebuilder:validation:Enum=idxd;vfio-pci
Comment thread
mythi marked this conversation as resolved.
// +kubebuilder:default=idxd
Driver string `json:"driver,omitempty"`

// Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint.
Tolerations []v1.Toleration `json:"tolerations,omitempty"`

Expand Down
Loading