chore: WIP - native sidecar fixes and test support

* There where we iterate over containers, also iterate over init containers
* Set `proxy.nativeSidecar: true`, for all tests to run in that mode without further changes
* Update golden files accordingly
* Fix `curl.rs` in the policy tests so it doesn't block on waiting for the proxy container to terminate
* Fix integration tests dealing with injection

Note that k8s started supporting native sidecars without additional feature flags in v1.28. For this reason the following tests aren't supposed to pass:

* test-policy with k8s v1.23
* test-multicluster with k8s v.1.23
* CNI integration test with k8s v1.27

Author: alpeb

charts/linkerd-control-plane/values.yaml

@@ -249,7 +249,7 @@ proxy:
   # -- Enable KEP-753 native sidecars
   # This is an experimental feature. It requires Kubernetes >= 1.29.
   # If enabled, .proxy.waitBeforeExitSeconds should not be used.
-  nativeSidecar: false
+  nativeSidecar: true
   # -- Native sidecar proxy startup probe parameters.
   # -- LivenessProbe timeout and delay configuration
   livenessProbe:

cli/cmd/inject_test.go

@@ -725,7 +725,6 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 	values.Proxy.Await = false
 	values.Proxy.AccessLog = "apache"
 	values.Proxy.ShutdownGracePeriod = "60s"
-	values.Proxy.NativeSidecar = true
 
 	expectedOverrides := map[string]string{
 		k8s.ProxyIgnoreInboundPortsAnnotation:  "8500-8505",
@@ -748,7 +747,6 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 		k8s.ProxyAwait:                            "disabled",
 		k8s.ProxyAccessLogAnnotation:              "apache",
 		k8s.ProxyShutdownGracePeriodAnnotation:    "60s",
-		k8s.ProxyEnableNativeSidecarAnnotation:    "true",
 	}
 
 	overrides := getOverrideAnnotations(values, baseValues)
@@ -877,6 +875,7 @@ func TestOverwriteRegistry(t *testing.T) {
 }
 
 func diffOverrides(t *testing.T, expectedOverrides map[string]string, actualOverrides map[string]string) {
+	t.Helper()
 	if len(expectedOverrides) != len(actualOverrides) {
 		t.Fatalf("expected annotations:\n%s\nbut received:\n%s", expectedOverrides, actualOverrides)
 	}

cli/cmd/testdata/inject-filepath/expected/injected_nginx.yaml

@@ -19,6 +19,48 @@ spec:
         linkerd.io/workload-ns: ""
     spec:
       containers:
+      - image: nginx
+        name: nginx
+        ports:
+        - containerPort: 80
+          name: http
+      initContainers:
+      - args:
+        - --firewall-bin-path
+        - iptables-nft
+        - --firewall-save-bin-path
+        - iptables-nft-save
+        - --ipv6=false
+        - --incoming-proxy-port
+        - "4143"
+        - --outgoing-proxy-port
+        - "4140"
+        - --proxy-uid
+        - "2102"
+        - --inbound-ports-to-ignore
+        - 4190,4191,4567,4568
+        - --outbound-ports-to-ignore
+        - 4567,4568
+        image: cr.l5d.io/linkerd/proxy-init:v2.4.3
+        imagePullPolicy: IfNotPresent
+        name: linkerd-init
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+          privileged: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 65534
+          runAsNonRoot: true
+          runAsUser: 65534
+          seccompProfile:
+            type: RuntimeDefault
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /run
+          name: linkerd-proxy-init-xtables-lock
       - env:
         - name: _pod_name
           valueFrom:
@@ -158,13 +200,6 @@ spec:
           value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
         image: cr.l5d.io/linkerd/proxy:install-proxy-version
         imagePullPolicy: IfNotPresent
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - /usr/lib/linkerd/linkerd-await
-              - --timeout=2m
-              - --port=4191
         livenessProbe:
           httpGet:
             path: /live
@@ -183,61 +218,26 @@ spec:
             port: 4191
           initialDelaySeconds: 2
           timeoutSeconds: 1
+        restartPolicy: Always
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
           runAsNonRoot: true
           runAsUser: 2102
           seccompProfile:
             type: RuntimeDefault
+        startupProbe:
+          failureThreshold: 120
+          httpGet:
+            path: /ready
+            port: 4191
+          periodSeconds: 1
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - mountPath: /var/run/linkerd/identity/end-entity
           name: linkerd-identity-end-entity
         - mountPath: /var/run/secrets/tokens
           name: linkerd-identity-token
-      - image: nginx
-        name: nginx
-        ports:
-        - containerPort: 80
-          name: http
-      initContainers:
-      - args:
-        - --firewall-bin-path
-        - iptables-nft
-        - --firewall-save-bin-path
-        - iptables-nft-save
-        - --ipv6=false
-        - --incoming-proxy-port
-        - "4143"
-        - --outgoing-proxy-port
-        - "4140"
-        - --proxy-uid
-        - "2102"
-        - --inbound-ports-to-ignore
-        - 4190,4191,4567,4568
-        - --outbound-ports-to-ignore
-        - 4567,4568
-        image: cr.l5d.io/linkerd/proxy-init:v2.4.3
-        imagePullPolicy: IfNotPresent
-        name: linkerd-init
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-          privileged: false
-          readOnlyRootFilesystem: true
-          runAsGroup: 65534
-          runAsNonRoot: true
-          runAsUser: 65534
-          seccompProfile:
-            type: RuntimeDefault
-        terminationMessagePolicy: FallbackToLogsOnError
-        volumeMounts:
-        - mountPath: /run
-          name: linkerd-proxy-init-xtables-lock
       volumes:
       - emptyDir: {}
         name: linkerd-proxy-init-xtables-lock