Feature: Add Istio Ambient Mode Support via Overlay Method (#3246)

* feature: ambient with overlay method

Signed-off-by: madmecodes <[email protected]>

* update: remove empty line minor formatting issue

Signed-off-by: madmecodes <[email protected]>

* update: remove duplication and use inheritance for gke overlay in ambient

Signed-off-by: madmecodes <[email protected]>

* update: remove common env variables already in base

Signed-off-by: madmecodes <[email protected]>

* feat: using components approach to enable ambient-mode and gke specific ambient-gke

Signed-off-by: madmecodes <[email protected]>

* added ambient workflow to test in kind cluster

Signed-off-by: madmecodes <[email protected]>

* feat: add Istio Ambient Mode with PSS baseline requirement

Signed-off-by: madmecodes <[email protected]>

* update: ambient pss upgraded to privilidged

Signed-off-by: madmecodes <[email protected]>

* upate: cni ztunnel connection debug

Signed-off-by: madmecodes <[email protected]>

* fix: enable ambient mode in CNI

Signed-off-by: madmecodes <[email protected]>

* fix: workflow updates from baseline to privilidged

Signed-off-by: madmecodes <[email protected]>

* fix: ambient netns mount, rbac fix

Signed-off-by: madmecodes <[email protected]>

* fix: ambient netns mount, rbac fix2

Signed-off-by: madmecodes <[email protected]>

* fix: remove duplication for ambient-oauth2-proxy

Signed-off-by: madmecodes <[email protected]>

* update: workflow added in matrix

Signed-off-by: madmecodes <[email protected]>

* fix: workflow matrix fix order

Signed-off-by: madmecodes <[email protected]>

* update: gke-cni component created and remove duplicacy

Signed-off-by: madmecodes <[email protected]>

* fix: corrected file name from baseline to privileged

Signed-off-by: madmecodes <[email protected]>

* fix: remove unwanted echo

Signed-off-by: madmecodes <[email protected]>

* Update common/istio/README.md

Signed-off-by: Julius von Kohout <[email protected]>

* Delete common/istio/istio-install/components/ambient-mode/istio-namespace-pss-privilege.yaml

Signed-off-by: Julius von Kohout <[email protected]>

* Update common/istio/README.md

Signed-off-by: Julius von Kohout <[email protected]>

* Update kustomization.yaml

Signed-off-by: Julius von Kohout <[email protected]>

---------

Signed-off-by: madmecodes <[email protected]>
Signed-off-by: Julius von Kohout <[email protected]>
Co-authored-by: Julius von Kohout <[email protected]>

Author: madmecodes

.github/workflows/istio_validation.yaml

@@ -1,11 +1,12 @@
-name: Test Istio CNI and Non-CNI Installation
+name: Test Istio Installation Modes (CNI, Insecure, Ambient)
 
 on:
   pull_request:
     paths:
     - tests/install_KinD_create_KinD_cluster_install_kustomize.sh
     - tests/istio*
-    - .github/workflows/istio_cni_test.yaml
+    - tests/ambient*
+    - .github/workflows/istio_validation.yaml
     - common/istio/**
     - common/cert-manager/**
 
@@ -21,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        istio-mode: ['cni', 'insecure']
+        istio-mode: ['cni', 'insecure', 'ambient']
 
     steps:
     - name: Checkout
@@ -61,6 +62,14 @@ jobs:
         kubectl patch validatingwebhookconfiguration istio-validator-istio-system --type='json' \
           -p="[{'op': 'add', 'path': '/webhooks/0/clientConfig/caBundle', 'value':'$CA_BUNDLE'}]"
 
+    - name: Install Istio Ambient Mode
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        kustomize build common/istio/istio-crds/base | kubectl apply -f -
+        kustomize build common/istio/istio-namespace/base | kubectl apply -f -
+        kustomize build common/istio/istio-install/overlays/ambient | kubectl apply -f -
+        kubectl wait --for=condition=Ready pods --all -n istio-system --timeout 300s
+
     - name: Wait for Istio deployment
       run: |
         kubectl wait --for=condition=available --timeout=300s deployment/istiod -n istio-system
@@ -92,6 +101,30 @@ jobs:
           (echo "ERROR: pilot.cni.enabled should be false in configmap values" && \
            kubectl get configmap istio-sidecar-injector -n istio-system -o jsonpath='{.data.values}' && exit 1)
 
+    - name: Verify Ambient Mode Installation
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        kubectl get daemonset ztunnel -n istio-system
+        kubectl rollout status daemonset/ztunnel -n istio-system --timeout=120s
+
+        PILOT_ENABLE_AMBIENT=$(kubectl get deployment istiod -n istio-system -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="PILOT_ENABLE_AMBIENT")].value}')
+        if [ "$PILOT_ENABLE_AMBIENT" != "true" ]; then
+          echo "ERROR: PILOT_ENABLE_AMBIENT should be true, got: $PILOT_ENABLE_AMBIENT"
+          exit 1
+        fi
+        echo "Ambient mode enabled in istiod"
+
+        kubectl get daemonset istio-cni-node -n kube-system
+        kubectl rollout status daemonset/istio-cni-node -n kube-system --timeout=120s
+        echo "CNI installed for ambient mode"
+
+        PSS_LEVEL=$(kubectl get namespace istio-system -o jsonpath='{.metadata.labels.pod-security\.kubernetes\.io/enforce}')
+        if [ "$PSS_LEVEL" != "privileged" ]; then
+          echo "ERROR: istio-system should have PSS privileged, got: $PSS_LEVEL"
+          exit 1
+        fi
+        echo "PSS privileged verified for istio-system"
+
     - name: Install OAuth2 Proxy
       run: ./tests/oauth2-proxy_install.sh
 
@@ -104,6 +137,12 @@ jobs:
     - name: Create KF Profile
       run: ./tests/kubeflow_profile_install.sh
 
+    - name: Enable ambient mode for user namespace
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        kubectl label namespace $KF_PROFILE istio-injection- --overwrite
+        kubectl label namespace $KF_PROFILE istio.io/dataplane-mode=ambient --overwrite
+
     - name: Enable Istio injection and adjust PSS for insecure mode
       run: |
 
@@ -133,14 +172,13 @@ jobs:
         EOF
 
     - name: Test sidecar injection
+      if: matrix.istio-mode != 'ambient'
       run: |
         kubectl apply -f ./tests/istio_dummy_deployment.yaml -n $KF_PROFILE
         kubectl wait --for=condition=available --timeout=160s deployment/test-application -n $KF_PROFILE
 
-        # Check for istio-proxy sidecar container injection
         POD_NAME=$(kubectl get pods -n $KF_PROFILE -l app=test-application -o jsonpath='{.items[0].metadata.name}')
 
-        # Check if istio-proxy container exists (either as sidecar or init container)
         ISTIO_PROXY_EXISTS=$(kubectl get pod $POD_NAME -n $KF_PROFILE -o jsonpath='{.spec.containers[*].name}' | grep -c "istio-proxy" || echo "0")
         ISTIO_INIT_EXISTS=$(kubectl get pod $POD_NAME -n $KF_PROFILE -o jsonpath='{.spec.initContainers[*].name}' | grep -c "istio-proxy" || echo "0")
 
@@ -150,12 +188,38 @@ jobs:
           kubectl get namespace $KF_PROFILE --show-labels
           exit 1
         fi
-        echo "Sidecar injection successful"
 
         # Verify Envoy proxy is responding
         kubectl exec -n $KF_PROFILE deployment/test-application -c istio-proxy -- curl -s localhost:15000/ready | grep -q "LIVE" || \
           (echo "ERROR: Envoy proxy not responding" && exit 1)
-        echo "Envoy proxy is healthy"
+
+    - name: Test ambient mode (no sidecar)
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        kubectl apply -f ./tests/istio_dummy_deployment.yaml -n $KF_PROFILE
+        kubectl wait --for=condition=available --timeout=160s deployment/test-application -n $KF_PROFILE
+
+        POD_NAME=$(kubectl get pods -n $KF_PROFILE -l app=test-application -o jsonpath='{.items[0].metadata.name}')
+
+        CONTAINER_COUNT=$(kubectl get pod $POD_NAME -n $KF_PROFILE -o jsonpath='{.spec.containers[*].name}' | wc -w)
+        if [ "$CONTAINER_COUNT" -ne "1" ]; then
+          echo "ERROR: Pod should have only 1 container in ambient mode, found: $CONTAINER_COUNT"
+          kubectl get pod $POD_NAME -n $KF_PROFILE -o jsonpath='{.spec.containers[*].name}'
+          exit 1
+        fi
+
+        AMBIENT_LABEL=$(kubectl get namespace $KF_PROFILE -o jsonpath='{.metadata.labels.istio\.io/dataplane-mode}')
+        if [ "$AMBIENT_LABEL" != "ambient" ]; then
+          echo "ERROR: Namespace should have istio.io/dataplane-mode=ambient label"
+          exit 1
+        fi
+
+    - name: Verify ztunnel logs show traffic handling
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        # Check ztunnel logs for workload processing
+        kubectl logs -n istio-system daemonset/ztunnel --tail=50 | grep -i "workload\|connection\|proxy" || echo "Ztunnel logs (last 50 lines):"
+        kubectl logs -n istio-system daemonset/ztunnel --tail=50
 
     - name: Test CNI-specific functionality
       if: matrix.istio-mode == 'cni'
@@ -179,8 +243,6 @@ jobs:
           exit 1
         fi
 
-        echo "Non-CNI verification successful: found istio-init, no istio-validation"
-
     - name: Port-forward the istio-ingress gateway
       run: ./tests/port_forward_gateway.sh
 
@@ -194,6 +256,13 @@ jobs:
       if: matrix.istio-mode == 'cni'
       run: ./tests/PSS_enable.sh
 
+    - name: Apply Pod Security Standards Restricted to user namespace (ambient mode)
+      if: matrix.istio-mode == 'ambient'
+      run: |
+        # Apply PSS restricted to user namespace (ztunnel runs in istio-system, not user namespaces)
+        kubectl label namespace $KF_PROFILE pod-security.kubernetes.io/enforce=restricted --overwrite
+        kubectl label namespace $KF_PROFILE pod-security.kubernetes.io/enforce-version=latest --overwrite
+
     - name: Collect debug information on failure
       if: failure()
       run: |
@@ -213,6 +282,10 @@ jobs:
         kubectl get daemonset istio-cni-node -n kube-system || echo "CNI daemonset not found (expected for insecure mode)"
         kubectl logs -n kube-system daemonset/istio-cni-node --tail=20 || true
 
+        echo "=== Ztunnel Daemonset Status ==="
+        kubectl get daemonset ztunnel -n istio-system || echo "Ztunnel daemonset not found (expected for non-ambient modes)"
+        kubectl logs -n istio-system daemonset/ztunnel --tail=20 || true
+
         echo "=== Webhook Configurations ==="
         kubectl get mutatingwebhookconfiguration istio-sidecar-injector -o yaml | grep -A5 -B5 caBundle || true
 

README.md

@@ -757,6 +757,6 @@ pre-commit run
 - **Q:** Why does Kubeflow use Istio CNI instead of standard Istio?
   **A:** Istio CNI provides better security by eliminating the need for privileged init containers, making it more compatible with Pod Security Standards (PSS). It also enables native sidecars support introduced in Kubernetes 1.28, which helps address issues with init containers and application lifecycle management.
 - **Q:** Why does Istio CNI fail on Google Kubernetes Engine (GKE) with "read-only file system" errors?
-  **A:** GKE mounts `/opt/cni/bin` as read-only for security reasons, preventing the Istio CNI installer from writing the CNI binary. Use the GKE-specific overlay: `kubectl apply -k common/istio/istio-install/overlays/gke`. This overlay uses GKE's writable CNI directory at `/home/kubernetes/bin`. For more details, see [Istio CNI Prerequisites](https://istio.io/latest/docs/setup/additional-setup/cni/#prerequisites) and [Platform Prerequisites](https://istio.io/latest/docs/ambient/install/platform-prerequisites/).-`
+  **A:** GKE mounts `/opt/cni/bin` as read-only for security reasons. Use the GKE-specific overlay: `kubectl apply -k common/istio/istio-install/overlays/gke` (or `overlays/ambient-gke` for ambient mode). These overlays use GKE's writable CNI directory at `/home/kubernetes/bin`. For details, see [Istio CNI Prerequisites](https://istio.io/latest/docs/setup/additional-setup/cni/#prerequisites).
 
 

common/istio/README.md

@@ -19,6 +19,21 @@ GKE mounts `/opt/cni/bin` as read-only for security reasons, preventing the Isti
 kubectl apply -k istio-install/overlays/gke
 ```
 
+### Ambient Mode (Sidecar-free Service Mesh)
+Istio Ambient Mode eliminates sidecars, reducing resource overhead while maintaining full L4/L7 traffic processing capabilities.
+
+```bash
+# OAuth2-Proxy
+kubectl apply -k istio-install/overlays/ambient-oauth2-proxy
+
+# OAuth2-Proxy on Google Kubernetes Engine (GKE)
+kubectl apply -k istio-install/overlays/ambient-oauth2-proxy-gke
+```
+
+**Important:** Ambient mode requires PSS Privileged (not Baseline or Restricted) for the `istio-system` namespace. The ztunnel component needs `CAP_SYS_ADMIN`, `CAP_NET_ADMIN`, and `CAP_NET_RAW` capabilities for transparent proxying and network namespace operations. The `istio-system` namespace is automatically configured with PSS privileged label when using ambient mode components.
+
+**Note:** Ambient mode uses Kustomize components (`components/ambient-mode/`) for composable configuration without duplication.
+
 ### Insecure Istio (CNI-disabled)
 For environments that don't support CNI:
 ```bash

common/istio/istio-install/components/ambient-mode/cni-ambient-patch.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: istio-cni-node
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: install-cni
+        volumeMounts:
+        - mountPath: /var/run/ztunnel
+          name: cni-ztunnel-sock-dir
+        - mountPath: /host/var/run/netns
+          name: cni-netns-dir
+          mountPropagation: HostToContainer
+      volumes:
+      - name: cni-ztunnel-sock-dir
+        hostPath:
+          path: /var/run/ztunnel
+          type: DirectoryOrCreate

common/istio/istio-install/components/ambient-mode/cni-config-patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: istio-cni-config
+  namespace: kube-system
+data:
+  AMBIENT_ENABLED: "true"

common/istio/istio-install/components/ambient-mode/cni-rbac-patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: istio-cni
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/status
+  - nodes
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update

common/istio/istio-install/components/ambient-mode/istio-namespace-pss-privileged.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: istio-system
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

common/istio/istio-install/components/ambient-mode/istiod-ambient-patch.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: istiod
+  namespace: istio-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: discovery
+        env:
+        - name: PILOT_ENABLE_AMBIENT
+          value: "true"

common/istio/istio-install/components/ambient-mode/kustomization.yaml

@@ -0,0 +1,34 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+- ztunnel.yaml
+- istio-namespace-pss-privileged.yaml
+
+patches:
+- path: istiod-ambient-patch.yaml
+  target:
+    kind: Deployment
+    name: istiod
+    namespace: istio-system
+- path: cni-ambient-patch.yaml
+  target:
+    kind: DaemonSet
+    name: istio-cni-node
+    namespace: kube-system
+- path: cni-config-patch.yaml
+  target:
+    kind: ConfigMap
+    name: istio-cni-config
+    namespace: kube-system
+- path: cni-rbac-patch.yaml
+  target:
+    kind: ClusterRole
+    name: istio-cni
+- target:
+    kind: Namespace
+    name: kubeflow
+  patch: |-
+    - op: add
+      path: /metadata/labels/istio.io~1dataplane-mode
+      value: ambient