feat(handler): auto-detect NetworkManager and use nmstatectl kernel mode

When NetworkManager is not available (e.g., kind containers, minimal hosts),
the handler now auto-detects this at startup and transparently switches to
nmstatectl kernel mode (-k flag). This affects both applying network config
(NNCP) and reporting state (NNS).

Key changes:
- Add kernelMode flag to nmstatectl package with -k flag for show/apply
- Skip checkpoint/rollback/commit/probes in kernel mode (not supported)
- Auto-detect NM absence at handler startup via nm.Version()
- Guard NM version reporting in node controller
- Update liveness probe to use -k flag when kernel-mode file exists
- Add init container to ensure dbus socket path exists on nodes without dbus
- Add feature Labels to all e2e handler tests for mode-based filtering
- Make BeforeSuite work in kernel mode (no DHCP reset)
- Add test-e2e-handler-kernel Makefile target with label filter
- Add GitHub Actions CI workflow for kernel mode e2e on kind

Kernel mode limitations: no checkpoint/rollback, no DHCP, volatile changes,
cannot create interfaces via nmstatectl apply -k (nispor plugin limitation).

Signed-off-by: Enrique Llorente <ellorent@redhat.com>

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Enrique Llorente <ellorent@redhat.com>

Author: qinqon

.github/workflows/e2e-kernel-mode.yml

@@ -0,0 +1,76 @@
+name: E2E Kernel Mode
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+jobs:
+  e2e-kernel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Write kind config
+        run: |
+          cat > /tmp/kind-config.yaml <<'EOF'
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          containerdConfigPatches:
+            - |-
+              [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
+                endpoint = ["http://kind-registry:5000"]
+          nodes:
+            - role: control-plane
+            - role: worker
+            - role: worker
+          EOF
+
+      - name: Start local registry
+        run: |
+          docker run -d --restart=always -p 5000:5000 --name kind-registry registry:2
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: kernel-mode-test
+          node_image: kindest/node:v1.32.0
+          config: /tmp/kind-config.yaml
+
+      - name: Connect registry to kind network
+        run: |
+          docker network connect kind kind-registry || true
+
+      - name: Create secondary docker networks and attach to kind nodes
+        run: |
+          docker network create --driver bridge kind-secondary-1
+          docker network create --driver bridge kind-secondary-2
+          for node in $(kind get nodes --name kernel-mode-test); do
+            docker network connect kind-secondary-1 "$node"
+            docker network connect kind-secondary-2 "$node"
+          done
+
+      - name: Label kind worker nodes
+        run: |
+          for node in $(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | grep worker); do
+            kubectl label node "$node" node-role.kubernetes.io/worker="" --overwrite
+          done
+
+      - name: Deploy with cluster-sync
+        run: |
+          make cluster-up
+          make cluster-sync
+        env:
+          KUBEVIRT_PROVIDER: external
+          KUBECONFIG: /home/runner/.kube/config
+          DEV_IMAGE_REGISTRY: localhost:5000
+          IMAGE_BUILDER: docker
+          KUBEVIRT_NUM_NODES: 3
+
+      - name: Run kernel mode e2e tests
+        run: make test-e2e-handler-kernel
+        env:
+          KUBEVIRT_PROVIDER: external
+          KUBECONFIG: /home/runner/.kube/config

Makefile

@@ -205,9 +205,18 @@ test-reporter:
 		go run . --dry-run --fake=failure && \
 		go run . --dry-run --fake=stale
 
+# Features supported in kernel mode (no NetworkManager).
+# Only read-only NNS tests — nmstatectl kernel mode (-k) cannot create
+# interfaces via apply (nispor plugin limitation).
+KERNEL_MODE_FEATURES ?= Nodes || NNSDependencies
+
 test-e2e-handler:
 	KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) $(GINKGO) $(e2e_test_args) ./test/e2e/handler ...
 
+test-e2e-handler-kernel:
+	KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) \
+		$(GINKGO) $(e2e_test_args) --label-filter="$(KERNEL_MODE_FEATURES)" ./test/e2e/handler ...
+
 test-e2e-operator: manifests
 	KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) $(GINKGO) $(e2e_test_args) ./test/e2e/operator ...
 
@@ -286,6 +295,7 @@ olm-push: bundle-push index-push
 	check-gen \
 	operator-sdk \
 	test-e2e-handler \
+	test-e2e-handler-kernel \
 	test-e2e-operator \
 	test-e2e \
 	test-reporter\

cluster/up.sh

@@ -4,26 +4,31 @@ set -ex
 
 source ./cluster/kubevirtci.sh
 kubevirtci::install
+source ./cluster/sync-common.sh
 
 $(kubevirtci::path)/cluster-up/up.sh
 
 if [[ "$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \
 		while ! $(KUBECTL) get securitycontextconstraints; do sleep 1; done; \
 fi
 
-echo 'Upgrading NetworkManager and enabling and starting up openvswitch'
-for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do
-    if [[ "$NM_VERSION" == "latest" ]]; then
-        echo "Installing NetworkManager from copr networkmanager/NetworkManager-main"
-        ./cluster/cli.sh ssh ${node} -- sudo dnf install -y dnf-plugins-core
-        ./cluster/cli.sh ssh ${node} -- sudo dnf copr enable -y networkmanager/NetworkManager-main
-    fi
-    ./cluster/cli.sh ssh ${node} -- sudo dnf upgrade -y NetworkManager --allowerasing
-    ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload
-    ./cluster/cli.sh ssh ${node} -- sudo systemctl enable openvswitch
-    ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch
-    # Newer kubevirtci has dhclient installed so we should enforce not using it to
-    # keep using the NM internal DHCP client as we always have
-    ./cluster/cli.sh ssh ${node} -- sudo rm -f /etc/NetworkManager/conf.d/002-dhclient.conf
-    ./cluster/cli.sh ssh ${node} -- sudo systemctl restart NetworkManager
-done
+if isExternal; then
+    echo 'Skipping NetworkManager/OVS setup for external provider'
+else
+    echo 'Upgrading NetworkManager and enabling and starting up openvswitch'
+    for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do
+        if [[ "$NM_VERSION" == "latest" ]]; then
+            echo "Installing NetworkManager from copr networkmanager/NetworkManager-main"
+            ./cluster/cli.sh ssh ${node} -- sudo dnf install -y dnf-plugins-core
+            ./cluster/cli.sh ssh ${node} -- sudo dnf copr enable -y networkmanager/NetworkManager-main
+        fi
+        ./cluster/cli.sh ssh ${node} -- sudo dnf upgrade -y NetworkManager --allowerasing
+        ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload
+        ./cluster/cli.sh ssh ${node} -- sudo systemctl enable openvswitch
+        ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch
+        # Newer kubevirtci has dhclient installed so we should enforce not using it to
+        # keep using the NM internal DHCP client as we always have
+        ./cluster/cli.sh ssh ${node} -- sudo rm -f /etc/NetworkManager/conf.d/002-dhclient.conf
+        ./cluster/cli.sh ssh ${node} -- sudo systemctl restart NetworkManager
+    done
+fi

cmd/handler/main.go

@@ -63,6 +63,7 @@ import (
 	"github.com/nmstate/kubernetes-nmstate/pkg/file"
 	nmstatelog "github.com/nmstate/kubernetes-nmstate/pkg/log"
 	"github.com/nmstate/kubernetes-nmstate/pkg/monitoring"
+	"github.com/nmstate/kubernetes-nmstate/pkg/nm"
 	"github.com/nmstate/kubernetes-nmstate/pkg/nmstatectl"
 	"github.com/nmstate/kubernetes-nmstate/pkg/webhook"
 )
@@ -224,9 +225,29 @@ func setupWebhookEnvironment(mgr manager.Manager) error {
 	return nil
 }
 
+// detectAndSetKernelMode checks if NetworkManager is available and enables kernel mode if not.
+func detectAndSetKernelMode() {
+	_, err := nm.Version()
+	if err != nil {
+		setupLog.Info("NetworkManager not available, enabling kernel mode",
+			"error", err.Error())
+		nmstatectl.SetKernelMode(true)
+	} else {
+		setupLog.Info("NetworkManager detected, using standard mode")
+	}
+}
+
 // setupHandlerEnvironment cleans up unavailableNodeCounts after unexpected restart,
 // configures the handler controllers and performs health checks
 func setupHandlerEnvironment(mgr manager.Manager) error {
+	detectAndSetKernelMode()
+
+	if nmstatectl.IsKernelMode() {
+		if err := file.Touch("/tmp/kernel-mode"); err != nil {
+			setupLog.Error(err, "Failed to write kernel-mode flag file")
+		}
+	}
+
 	// Clean stale unavailable counts from node before starting controllers
 	// Prevents deadlock after unexpected cluster reboot where nodes were
 	// processing NNCP and left stale counts in etcd.

controllers/handler/node_controller.go

@@ -128,9 +128,14 @@ func (r *NodeReconciler) getDependencyVersions() *nmstate.DependencyVersions {
 		r.Log.Error(err, "failed retrieving handler nmstate version")
 	}
 
-	hostNetworkManagerVersion, err := nm.Version()
-	if err != nil {
-		r.Log.Error(err, "error retrieving host Networkmanager version")
+	var hostNetworkManagerVersion string
+	if nmstatectl.IsKernelMode() {
+		hostNetworkManagerVersion = "N/A (kernel mode)"
+	} else {
+		hostNetworkManagerVersion, err = nm.Version()
+		if err != nil {
+			r.Log.Error(err, "error retrieving host Networkmanager version")
+		}
 	}
 
 	return &nmstate.DependencyVersions{

controllers/operator/nmstate_controller_test.go

@@ -487,7 +487,7 @@ var _ = Describe("NMState controller reconcile", func() {
 				ds := &appsv1.DaemonSet{}
 				err := cl.Get(context.Background(), handlerKey, ds)
 				Expect(err).ToNot(HaveOccurred())
-				expectedCommand := "nmstatectl show -vv 2>&1"
+				expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k -vv 2>&1; else nmstatectl show -vv 2>&1; fi"
 				Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand))
 			})
 		})
@@ -515,7 +515,7 @@ var _ = Describe("NMState controller reconcile", func() {
 				ds := &appsv1.DaemonSet{}
 				err := cl.Get(context.Background(), handlerKey, ds)
 				Expect(err).ToNot(HaveOccurred())
-				expectedCommand := "nmstatectl show  2>&1"
+				expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k  2>&1; else nmstatectl show  2>&1; fi"
 				Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand))
 			})
 		})
@@ -542,7 +542,7 @@ var _ = Describe("NMState controller reconcile", func() {
 				ds := &appsv1.DaemonSet{}
 				err := cl.Get(context.Background(), handlerKey, ds)
 				Expect(err).ToNot(HaveOccurred())
-				expectedCommand := "nmstatectl show  2>&1"
+				expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k  2>&1; else nmstatectl show  2>&1; fi"
 				Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand))
 			})
 		})

deploy/handler/operator.yaml

@@ -361,6 +361,19 @@ spec:
       tolerations: {{ toYaml .HandlerTolerations | nindent 8 }}
       affinity: {{ toYaml .HandlerAffinity | nindent 8 }}
       priorityClassName: system-node-critical
+      initContainers:
+        - name: ensure-dbus-socket
+          image: {{ .HandlerImage }}
+          imagePullPolicy: {{ .HandlerPullPolicy }}
+          command:
+            - sh
+            - -c
+            - "mkdir -p /host-run/dbus && test -e /host-run/dbus/system_bus_socket || touch /host-run/dbus/system_bus_socket"
+          volumeMounts:
+            - name: host-run
+              mountPath: /host-run
+          securityContext:
+            privileged: true
       containers:
         - name: nmstate-handler
           args:
@@ -445,17 +458,19 @@ spec:
               command:
               - bash
               - -c
-              - "nmstatectl show {{ .HandlerReadinessProbeExtraArg }} 2>&1"
+              - "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k {{ .HandlerReadinessProbeExtraArg }} 2>&1; else nmstatectl show {{ .HandlerReadinessProbeExtraArg }} 2>&1; fi"
             initialDelaySeconds: 60
             periodSeconds: 60
             timeoutSeconds: 10
             successThreshold: 1
             failureThreshold: 5
       volumes:
+        - name: host-run
+          hostPath:
+            path: /run
         - name: dbus-socket
           hostPath:
             path: /run/dbus/system_bus_socket
-            type: Socket
         - name: nmstate-lock
           hostPath:
             path: /var/k8s_nmstate

pkg/client/client.go

@@ -154,6 +154,15 @@ func ApplyDesiredState(ctx context.Context, cli client.Client, desiredState shar
 		return "Ignoring empty desired state", nil
 	}
 
+	if nmstatectl.IsKernelMode() {
+		log.Info("Kernel mode: applying desired state without checkpoint/rollback/probes")
+		setOutput, err := nmstatectl.Set(desiredState, DesiredStateConfigurationTimeout)
+		if err != nil {
+			return setOutput, err
+		}
+		return fmt.Sprintf("setOutput: %s \n", setOutput), nil
+	}
+
 	// Before apply we get the probes that are working fine, they should be
 	// working fine after apply
 	probes := probe.Select(ctx, cli)

pkg/nmstatectl/nmstatectl.go

@@ -28,12 +28,20 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/yaml"
 
 	nmstate "github.com/nmstate/kubernetes-nmstate/api/shared"
 )
 
-var debugMode bool
+var (
+	debugMode  bool
+	kernelMode bool
+	log        = logf.Log.WithName("nmstatectl")
+)
+
+func SetKernelMode(kernel bool) { kernelMode = kernel }
+func IsKernelMode() bool        { return kernelMode }
 
 const nmstateCommand = "nmstatectl"
 
@@ -88,11 +96,20 @@ func nmstatectl(arguments []string) (string, error) {
 }
 
 func ShowWithArgumentsAndOutputs(arguments []string, stdout, stderr io.Writer) error {
-	return nmstatectlWithInputAndOutputs(append([]string{"show"}, arguments...), "", stdout, stderr)
+	args := []string{"show"}
+	if kernelMode {
+		args = append(args, "-k")
+	}
+	args = append(args, arguments...)
+	return nmstatectlWithInputAndOutputs(args, "", stdout, stderr)
 }
 
 func Show() (string, error) {
-	return nmstatectl([]string{"show"})
+	args := []string{"show"}
+	if kernelMode {
+		args = append(args, "-k")
+	}
+	return nmstatectl(args)
 }
 
 func Set(desiredState nmstate.State, timeout time.Duration) (string, error) {
@@ -103,17 +120,30 @@ func Set(desiredState nmstate.State, timeout time.Duration) (string, error) {
 	if debugMode {
 		args = append(args, "-vv")
 	}
-	args = append(args, "--no-commit", "--timeout", strconv.Itoa(int(timeout.Seconds())))
+	if kernelMode {
+		log.Info("Kernel mode: applying with -k flag, skipping --no-commit and --timeout (checkpoints not supported)")
+		args = append(args, "-k")
+	} else {
+		args = append(args, "--no-commit", "--timeout", strconv.Itoa(int(timeout.Seconds())))
+	}
 
 	setOutput, err := nmstatectlWithInput(args, string(desiredState.Raw))
 	return setOutput, err
 }
 
 func Commit() (string, error) {
+	if kernelMode {
+		log.Info("Kernel mode: skipping commit (checkpoints not supported)")
+		return "commit skipped (kernel mode)", nil
+	}
 	return nmstatectl([]string{"commit"})
 }
 
 func Rollback() error {
+	if kernelMode {
+		log.Info("Kernel mode: skipping rollback (checkpoints not supported)")
+		return nil
+	}
 	_, err := nmstatectl([]string{"rollback"})
 	if err != nil {
 		return errors.Wrapf(err, "failed calling nmstatectl rollback")