diff --git a/README.md b/README.md index 4be8588..6523951 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,8 @@ kind: Pod metadata: name: my-hyperlight-app spec: - runtimeClassName: hyperlight-kvm # or hyperlight-mshv + nodeSelector: + hyperlight.dev/hypervisor: kvm # or mshv automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/deploy/azure/setup.sh b/deploy/azure/setup.sh index 6d9b147..d45c95e 100755 --- a/deploy/azure/setup.sh +++ b/deploy/azure/setup.sh @@ -157,7 +157,6 @@ create_kvm_nodepool() { --min-count "${KVM_NODE_MIN_COUNT}" \ --max-count "${KVM_NODE_MAX_COUNT}" \ --labels "hyperlight.dev/hypervisor=kvm" "hyperlight.dev/enabled=true" \ - --node-taints "hyperlight.dev/hypervisor=kvm:NoSchedule" \ --mode User \ --ssh-access disabled \ -o none @@ -184,7 +183,6 @@ create_mshv_nodepool() { --min-count "${MSHV_NODE_MIN_COUNT}" \ --max-count "${MSHV_NODE_MAX_COUNT}" \ --labels "hyperlight.dev/hypervisor=mshv" "hyperlight.dev/enabled=true" \ - --node-taints "hyperlight.dev/hypervisor=mshv:NoSchedule" \ --mode User \ --ssh-access disabled \ -o none diff --git a/deploy/local/device-plugin.yaml b/deploy/local/device-plugin.yaml index 82f5b7c..f7ddf89 100644 --- a/deploy/local/device-plugin.yaml +++ b/deploy/local/device-plugin.yaml @@ -24,44 +24,6 @@ metadata: app.kubernetes.io/name: hyperlight-system app.kubernetes.io/part-of: hyperlight ---- -# RuntimeClass for KVM-based Hyperlight workloads -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: hyperlight-kvm - labels: - app.kubernetes.io/name: hyperlight-kvm - app.kubernetes.io/part-of: hyperlight -handler: runc -scheduling: - nodeSelector: - hyperlight.dev/hypervisor: kvm - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "kvm" - effect: "NoSchedule" - ---- -# RuntimeClass for MSHV-based Hyperlight workloads -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: hyperlight-mshv - labels: - app.kubernetes.io/name: hyperlight-mshv - app.kubernetes.io/part-of: hyperlight -handler: runc -scheduling: - nodeSelector: - hyperlight.dev/hypervisor: mshv - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "mshv" - effect: "NoSchedule" - --- # ServiceAccount for device plugin apiVersion: v1 @@ -99,11 +61,6 @@ spec: # Only run on nodes with hyperlight enabled nodeSelector: hyperlight.dev/enabled: "true" - # Tolerate hypervisor taints - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Exists" - effect: "NoSchedule" priorityClassName: system-node-critical containers: - name: device-plugin diff --git a/deploy/local/setup.sh b/deploy/local/setup.sh index c08055b..a3a239d 100755 --- a/deploy/local/setup.sh +++ b/deploy/local/setup.sh @@ -98,7 +98,7 @@ EOF } setup_node_labels() { - log_info "Setting up node labels and taints..." + log_info "Setting up node labels..." # The labels are set in kind-config.yaml, but let's ensure they exist local node @@ -107,9 +107,6 @@ setup_node_labels() { kubectl label node "${node}" hyperlight.dev/enabled=true --overwrite kubectl label node "${node}" hyperlight.dev/hypervisor=kvm --overwrite - # Add taint (optional for local dev, but matches production) - kubectl taint node "${node}" hyperlight.dev/hypervisor=kvm:NoSchedule --overwrite 2>/dev/null || true - log_success "Node configured" } diff --git a/deploy/manifests/device-plugin.yaml b/deploy/manifests/device-plugin.yaml index a4cf85c..8f15bec 100644 --- a/deploy/manifests/device-plugin.yaml +++ b/deploy/manifests/device-plugin.yaml @@ -8,44 +8,6 @@ metadata: app.kubernetes.io/name: hyperlight-system app.kubernetes.io/part-of: hyperlight ---- -# RuntimeClass for KVM-based Hyperlight workloads -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: hyperlight-kvm - labels: - app.kubernetes.io/name: hyperlight-kvm - app.kubernetes.io/part-of: hyperlight -handler: runc -scheduling: - nodeSelector: - hyperlight.dev/hypervisor: kvm - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "kvm" - effect: "NoSchedule" - ---- -# RuntimeClass for MSHV-based Hyperlight workloads -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: hyperlight-mshv - labels: - app.kubernetes.io/name: hyperlight-mshv - app.kubernetes.io/part-of: hyperlight -handler: runc -scheduling: - nodeSelector: - hyperlight.dev/hypervisor: mshv - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "mshv" - effect: "NoSchedule" - --- # ServiceAccount for device plugin apiVersion: v1 @@ -83,11 +45,6 @@ spec: # Only run on nodes with hyperlight enabled nodeSelector: hyperlight.dev/enabled: "true" - # Tolerate hypervisor taints - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Exists" - effect: "NoSchedule" priorityClassName: system-node-critical containers: - name: device-plugin diff --git a/deploy/manifests/examples/deployment-kvm.yaml b/deploy/manifests/examples/deployment-kvm.yaml index de9deb8..13d9361 100644 --- a/deploy/manifests/examples/deployment-kvm.yaml +++ b/deploy/manifests/examples/deployment-kvm.yaml @@ -21,8 +21,9 @@ spec: app.kubernetes.io/name: hyperlight-app hyperlight.dev/hypervisor: kvm spec: - runtimeClassName: hyperlight-kvm - # Disable K8s API access - most apps don't need it + nodeSelector: + hyperlight.dev/hypervisor: kvm + # Disable K8s API access automountServiceAccountToken: false # Explicitly disable host namespaces hostNetwork: false diff --git a/deploy/manifests/examples/deployment-mshv.yaml b/deploy/manifests/examples/deployment-mshv.yaml index 3719df6..0d8e62d 100644 --- a/deploy/manifests/examples/deployment-mshv.yaml +++ b/deploy/manifests/examples/deployment-mshv.yaml @@ -21,8 +21,9 @@ spec: app.kubernetes.io/name: hyperlight-app hyperlight.dev/hypervisor: mshv spec: - runtimeClassName: hyperlight-mshv - # Disable K8s API access - most apps don't need it + nodeSelector: + hyperlight.dev/hypervisor: mshv + # Disable K8s API access automountServiceAccountToken: false # Explicitly disable host namespaces hostNetwork: false diff --git a/deploy/manifests/examples/test-pod-kvm.yaml b/deploy/manifests/examples/test-pod-kvm.yaml index f79e072..cb950b8 100644 --- a/deploy/manifests/examples/test-pod-kvm.yaml +++ b/deploy/manifests/examples/test-pod-kvm.yaml @@ -9,7 +9,8 @@ metadata: app.kubernetes.io/name: hyperlight-test hyperlight.dev/hypervisor: kvm spec: - runtimeClassName: hyperlight-kvm + nodeSelector: + hyperlight.dev/hypervisor: kvm containers: - name: test image: alpine:3.19 diff --git a/deploy/manifests/examples/test-pod-mshv.yaml b/deploy/manifests/examples/test-pod-mshv.yaml index e361fa0..26cba52 100644 --- a/deploy/manifests/examples/test-pod-mshv.yaml +++ b/deploy/manifests/examples/test-pod-mshv.yaml @@ -9,7 +9,8 @@ metadata: app.kubernetes.io/name: hyperlight-test hyperlight.dev/hypervisor: mshv spec: - runtimeClassName: hyperlight-mshv + nodeSelector: + hyperlight.dev/hypervisor: mshv containers: - name: test image: alpine:3.19 diff --git a/device-plugin/Dockerfile b/device-plugin/Dockerfile index 2953516..e7462ef 100644 --- a/device-plugin/Dockerfile +++ b/device-plugin/Dockerfile @@ -17,8 +17,7 @@ RUN apk --no-cache add ca-certificates COPY --from=builder /app/hyperlight-device-plugin /usr/local/bin/ -# Run as non-root -RUN adduser -D -u 1000 hyperlight -USER hyperlight +# Note: Runs as root (runAsUser: 0) in Kubernetes - required for +# write access to /var/lib/kubelet/device-plugins and /var/run/cdi ENTRYPOINT ["hyperlight-device-plugin"] diff --git a/docs/architecture.md b/docs/architecture.md index 2744390..14783b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -177,37 +177,35 @@ The CDI spec sets device uid/gid to match the container user: This allows the non-root container process to access the hypervisor device. -## RuntimeClasses +## Scheduling -Two RuntimeClasses route pods to appropriate node pools: +Pods target specific hypervisors using `nodeSelector` and get device access via resource requests: -| RuntimeClass | Hypervisor | Node Selector | Toleration | -|--------------|------------|---------------|------------| -| `hyperlight-kvm` | KVM | `hyperlight.dev/hypervisor=kvm` | `hyperlight.dev/hypervisor=kvm:NoSchedule` | -| `hyperlight-mshv` | MSHV | `hyperlight.dev/hypervisor=mshv` | `hyperlight.dev/hypervisor=mshv:NoSchedule` | - -Usage: ```yaml spec: - runtimeClassName: hyperlight-kvm + nodeSelector: + hyperlight.dev/hypervisor: kvm # or mshv + containers: + - name: app + resources: + limits: + hyperlight.dev/hypervisor: "1" ``` -## Node Labels and Taints +| Component | Purpose | +|-----------|---------| +| `nodeSelector` | Ensures pod lands on a node with the specified hypervisor | +| Resource request | Triggers CDI injection of `/dev/kvm` or `/dev/mshv` | + +## Node Labels -### Labels +The device plugin automatically labels nodes based on detected hypervisor: | Label | Values | Purpose | |-------|--------|---------| | `hyperlight.dev/enabled` | `true` | Device plugin runs on this node | | `hyperlight.dev/hypervisor` | `kvm` or `mshv` | Which hypervisor is available | -### Taints - -| Taint | Effect | Purpose | -|-------|--------|---------| -| `hyperlight.dev/hypervisor=kvm:NoSchedule` | NoSchedule | Only Hyperlight workloads on KVM nodes | -| `hyperlight.dev/hypervisor=mshv:NoSchedule` | NoSchedule | Only Hyperlight workloads on MSHV nodes | - ## Next Steps - [Local Development](local-development.md) - Test locally with KIND diff --git a/hyperlight-app/k8s/deployment-kvm.yaml b/hyperlight-app/k8s/deployment-kvm.yaml index 504c4fd..d1aa5f0 100644 --- a/hyperlight-app/k8s/deployment-kvm.yaml +++ b/hyperlight-app/k8s/deployment-kvm.yaml @@ -22,11 +22,6 @@ spec: spec: nodeSelector: hyperlight.dev/hypervisor: kvm - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "kvm" - effect: "NoSchedule" securityContext: runAsNonRoot: true runAsUser: 65534 diff --git a/hyperlight-app/k8s/deployment-mshv.yaml b/hyperlight-app/k8s/deployment-mshv.yaml index e77b6e5..e27e402 100644 --- a/hyperlight-app/k8s/deployment-mshv.yaml +++ b/hyperlight-app/k8s/deployment-mshv.yaml @@ -22,11 +22,6 @@ spec: spec: nodeSelector: hyperlight.dev/hypervisor: mshv - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "mshv" - effect: "NoSchedule" securityContext: runAsNonRoot: true runAsUser: 65534 diff --git a/hyperlight-app/k8s/deployment.yaml b/hyperlight-app/k8s/deployment.yaml index da7239e..a2ef93b 100644 --- a/hyperlight-app/k8s/deployment.yaml +++ b/hyperlight-app/k8s/deployment.yaml @@ -29,11 +29,6 @@ spec: hostIPC: false nodeSelector: hyperlight.dev/hypervisor: kvm - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "kvm" - effect: "NoSchedule" securityContext: runAsNonRoot: true runAsUser: 65534 @@ -92,11 +87,6 @@ spec: hostIPC: false nodeSelector: hyperlight.dev/hypervisor: mshv - tolerations: - - key: "hyperlight.dev/hypervisor" - operator: "Equal" - value: "mshv" - effect: "NoSchedule" securityContext: runAsNonRoot: true runAsUser: 65534