DRAAdminAccess: add example (#112)

* DRAAdminAccess: add example

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

* address comments and make it os agnostic

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

* address comments

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

* address comments

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

---------

Signed-off-by: Rita Zhang <rita.z.zhang@gmail.com>

Author: ritazh

README.md

@@ -348,10 +348,58 @@ You can use the IDs of the GPUs as well as the GPU sharing settings set in
 these environment variables to verify that they were handed out in a way
 consistent with the semantics shown in the figure above.
 
+### Demo DRA Admin Access Feature
+This example driver includes support for the [DRA AdminAccess feature](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#admin-access), which allows administrators to gain privileged access to devices already in use by other users. This example demonstrates the end-to-end flow by setting the `DRA_ADMIN_ACCESS` environment variable. A driver managing real devices could use this to expose host hardware information.
+
+#### Usage Example
+
+See `demo/gpu-test7.yaml` for a complete example. Key points:
+
+1. **Namespace**: Must have the `resource.kubernetes.io/admin-access` label set to create ResourceClaimTemplate and ResourceClaim with `adminAccess: true` for Kubernetes v1.34+.
+```yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gpu-test7
+  labels:
+    resource.kubernetes.io/admin-access: "true"
+```
+
+2. **Resource Claim Template**: Request must have `adminAccess: true`. The `allocationMode: All` is used to demonstrate accessing all available devices with admin privileges.
+```yaml
+spec:
+  spec:
+    devices:
+      requests:
+      - name: admin-gpu
+        exactly:
+          deviceClassName: gpu.example.com
+          allocationMode: All
+          adminAccess: true
+```
+
+3. **Container**: Will receive elevated privileges from the driver, represented here as environment variables
+```bash
+echo "DRA Admin Access: $DRA_ADMIN_ACCESS"
+# Output examples:
+# DRA Admin Access: true
+```
+
+#### Testing
+
+To run this demo:
+```bash
+./demo/test-admin-access.sh
+```
+
+This demonstration shows the end-to-end flow of the DRA AdminAccess feature. In a production environment, drivers could use this admin access indication to provide additional privileged capabilities or information to authorized workloads.
+
+### Clean Up
+
 Once you have verified everything is running correctly, delete all of the
 example apps:
 ```bash
-kubectl delete --wait=false --filename=demo/gpu-test{1,2,3,4,5}.yaml
+kubectl delete --wait=false --filename=demo/gpu-test{1,2,3,4,5,7}.yaml
 ```
 
 And wait for them to terminate:
@@ -366,6 +414,7 @@ gpu-test3   pod0   1/1     Terminating   0          31m
 gpu-test3   pod1   1/1     Terminating   0          31m
 gpu-test4   pod0   1/1     Terminating   0          31m
 gpu-test5   pod0   4/4     Terminating   0          31m
+gpu-test7   pod0   1/1     Terminating   0          31m
 ...
 ```
 

cmd/dra-example-kubeletplugin/cdi.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 The Kubernetes Authors.
+ * Copyright The Kubernetes Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -99,9 +99,13 @@ func (cdi *CDIHandler) CreateClaimSpecFile(claimUID string, devices profiles.Pre
 			ContainerEdits: &cdispec.ContainerEdits{
 				Env: []string{
 					fmt.Sprintf("%s_DEVICE_%s_RESOURCE_CLAIM=%s", strings.ToUpper(cdi.class), deviceEnvKey, claimUID),
+					fmt.Sprintf("DRA_ADMIN_ACCESS=%t", device.AdminAccess),
 				},
 			},
 		}
+
+		// If this device has admin access, then here is where to inject host hardware information
+
 		claimEdits.Append(device.ContainerEdits)
 
 		cdiDevice := cdispec.Device{

cmd/dra-example-kubeletplugin/driver.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 The Kubernetes Authors.
+ * Copyright The Kubernetes Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -95,8 +95,10 @@ func (d *driver) PrepareResourceClaims(ctx context.Context, claims []*resourceap
 }
 
 func (d *driver) prepareResourceClaim(_ context.Context, claim *resourceapi.ResourceClaim) kubeletplugin.PrepareResult {
+	klog.Infof("Preparing claim: UID=%s, Namespace=%s, Name=%s", claim.UID, claim.Namespace, claim.Name)
 	preparedPBs, err := d.state.Prepare(claim)
 	if err != nil {
+		klog.Errorf("Error preparing devices for claim %v: %v", claim.UID, err)
 		return kubeletplugin.PrepareResult{
 			Err: fmt.Errorf("error preparing devices for claim %v: %w", claim.UID, err),
 		}

cmd/dra-example-kubeletplugin/state.go

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 The Kubernetes Authors.
+ * Copyright The Kubernetes Authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -139,7 +139,6 @@ func (s *DeviceState) Prepare(claim *resourceapi.ResourceClaim) ([]*drapbv1.Devi
 	if preparedClaims[claimUID] != nil {
 		return preparedClaims[claimUID].GetDevices(), nil
 	}
-
 	preparedDevices, err := s.prepareDevices(claim)
 	if err != nil {
 		return nil, fmt.Errorf("prepare failed: %v", err)
@@ -163,7 +162,10 @@ func (s *DeviceState) Unprepare(claimUID string) error {
 
 	checkpoint := newCheckpoint()
 	if err := s.checkpointManager.GetCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil {
-		return fmt.Errorf("unable to sync from checkpoint: %v", err)
+		checkpoint = newCheckpoint()
+		if err := s.checkpointManager.CreateCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil {
+			return fmt.Errorf("unable to create new checkpoint: %v", err)
+		}
 	}
 	preparedClaims := checkpoint.V1.PreparedClaims
 
@@ -192,6 +194,8 @@ func (s *DeviceState) prepareDevices(claim *resourceapi.ResourceClaim) (profiles
 	if claim.Status.Allocation == nil {
 		return nil, fmt.Errorf("claim not yet allocated")
 	}
+	// Check if any device request has admin access
+	hasAdminAccess := s.checkAdminAccess(claim)
 
 	// Retrieve the full set of device configs for the driver.
 	configs, err := GetOpaqueDeviceConfigs(
@@ -257,6 +261,7 @@ func (s *DeviceState) prepareDevices(claim *resourceapi.ResourceClaim) (profiles
 					CdiDeviceIds: s.cdi.GetClaimDevices(string(claim.UID), []string{result.Device}),
 				},
 				ContainerEdits: perDeviceCDIContainerEdits[result.Device],
+				AdminAccess:    hasAdminAccess,
 			}
 			preparedDevices = append(preparedDevices, device)
 		}
@@ -269,6 +274,18 @@ func (s *DeviceState) unprepareDevices(claimUID string, devices profiles.Prepare
 	return nil
 }
 
+// checkAdminAccess determines if a resource claim requires admin access.
+func (s *DeviceState) checkAdminAccess(claim *resourceapi.ResourceClaim) bool {
+	if claim != nil && claim.Status.Allocation != nil {
+		for _, result := range claim.Status.Allocation.Devices.Results {
+			if result.AdminAccess != nil && *result.AdminAccess {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // GetOpaqueDeviceConfigs returns an ordered list of the configs contained in possibleConfigs for this driver.
 //
 // Configs can either come from the resource claim itself or from the device

demo/gpu-test7.yaml

@@ -0,0 +1,56 @@
+# One Namespace with admin access label
+# One pod with one container requesting all GPUs with admin access
+# This demo shows the DRA admin access feature with DRA_ADMIN_ACCESS environment variable
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gpu-test7
+  labels:
+    resource.kubernetes.io/admin-access: "true"
+---
+apiVersion: resource.k8s.io/v1
+kind: ResourceClaimTemplate
+metadata:
+  namespace: gpu-test7
+  name: multiple-gpus-admin
+spec:
+  spec: 
+    devices:
+      requests:
+      - name: admin-gpu
+        exactly:
+          deviceClassName: gpu.example.com
+          allocationMode: All
+          adminAccess: true
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: gpu-test7
+  name: pod0
+spec:
+  containers:
+  - name: ctr0
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args:
+    - |
+      export
+      echo "=== DRA Admin Access Demo ==="
+      echo "DRA Admin Access: $DRA_ADMIN_ACCESS"
+      echo ""
+      echo "GPU Environment Variables:"
+      env | grep GPU_ | sort
+      echo ""
+      echo "=== Sleeping to allow inspection ==="
+      trap 'exit 0' TERM
+      sleep 9999 & wait
+    resources:
+      claims:
+      - name: admin-gpus
+  resourceClaims:
+  - name: admin-gpus
+    resourceClaimTemplateName: multiple-gpus-admin

demo/scripts/kind-cluster-config.yaml

@@ -2,6 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 featureGates:
   DynamicResourceAllocation: true
+  DRAAdminAccess: true
 containerdConfigPatches:
 # Enable CDI as described in
 # https://tags.cncf.io/container-device-interface#containerd-configuration
@@ -17,11 +18,11 @@ nodes:
       extraArgs:
         runtime-config: "resource.k8s.io/v1beta1=true"
     scheduler:
-        extraArgs:
-          v: "1"
+      extraArgs:
+        v: "1"
     controllerManager:
-        extraArgs:
-          v: "1"
+      extraArgs:
+        v: "1"
   - |
     kind: InitConfiguration
     nodeRegistration:

demo/test-admin-access.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Copyright The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script demonstrates the DRA Admin Access feature by deploying
+# the demo and verifying the DRA_ADMIN_ACCESS environment variable is set
+
+set -e
+
+echo "=== DRA Admin Access Feature Test ==="
+echo
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo "❌ kubectl is not available. Please install kubectl and ensure cluster access."
+    exit 1
+fi
+
+# Check if the cluster is accessible
+if ! kubectl cluster-info &> /dev/null; then
+    echo "❌ Unable to access Kubernetes cluster. Please check your kubeconfig."
+    exit 1
+fi
+
+echo "✅ Kubernetes cluster is accessible"
+
+# Apply the demo
+echo "📦 Applying gpu-test7.yaml demo..."
+kubectl apply -f demo/gpu-test7.yaml
+
+echo "⏳ Waiting for pod to be ready..."
+kubectl wait --for=condition=Ready pod/pod0 -n gpu-test7 --timeout=120s || true
+
+echo
+echo "=== Pod Status ==="
+kubectl get pods -n gpu-test7
+
+echo
+echo "=== ResourceClaims Status ==="
+kubectl get resourceclaims -n gpu-test7
+
+echo
+echo "=== Pod0 Logs (showing admin access demo) ==="
+kubectl logs pod0 -n gpu-test7 || echo "⚠️  Pod0 logs not ready yet"
+
+echo
+echo "=== Checking DRA_ADMIN_ACCESS Environment Variable ==="
+DRA_ADMIN_ACCESS_POD0=$(kubectl exec pod0 -n gpu-test7 -- printenv DRA_ADMIN_ACCESS 2>/dev/null || echo "not found")
+
+if [[ "$DRA_ADMIN_ACCESS_POD0" == "true" ]]; then
+  echo "✅ Pod0: DRA_ADMIN_ACCESS=$DRA_ADMIN_ACCESS_POD0"
+else
+  echo "❌ Pod0: DRA_ADMIN_ACCESS=$DRA_ADMIN_ACCESS_POD0 (expected: true)"
+fi
+
+echo
+echo "=== Test Complete ==="
+echo "To clean up, run: kubectl delete namespace gpu-test7"

deployments/helm/dra-example-driver/values.yaml

@@ -37,7 +37,7 @@ serviceAccount:
 kubeletPlugin:
   # numDevices describes how many GPUs to advertise on each node when the "gpu"
   # deviceProfile is used. Not relevant for other profiles.
-  numDevices: 8
+  numDevices: 9
   priorityClassName: "system-node-critical"
   updateStrategy:
     type: RollingUpdate

internal/profiles/profiles.go

@@ -31,6 +31,7 @@ type PerDeviceCDIContainerEdits map[string]*cdiapi.ContainerEdits
 type PreparedDevice struct {
 	drapbv1.Device
 	ContainerEdits *cdiapi.ContainerEdits
+	AdminAccess    bool
 }
 
 type PreparedDevices []*PreparedDevice