test(policy): address timeouts and flakiness (#14773)

Policy tests are very flaky. Currently one of the main culprits is that service account creation sometimes isn't caught as an event by the watcher, blocking `await_service_account` until it times out after 60s. We already have in place up to 3 retries when calling `cargo nextest`, but these tests are sequential and the 60s timeouts start accumulating until we reach the CI job timeout at 20min.

This change first lowers the service account creation timeout down to 15s, understanding that if the watcher catches that event it will do pretty quickly or else block indefinitely. So better to fail faster and trigger the test retry ASAP.

With this change, `test-policy (v1.34, linkerd, experimental)` is finally passing, taking 17m due to the large number of retries:
```
     Summary [ 779.409s] 151 tests run: 151 passed (15 flaky), 0 skipped
   FLAKY 2/4 [   0.069s] linkerd-policy-test::admit_network_authentication rejects_invalid_cidr
   FLAKY 3/4 [  15.019s] linkerd-policy-test::e2e_audit ns_audit
   FLAKY 2/4 [  10.964s] linkerd-policy-test::e2e_authorization_policy targets_route
   FLAKY 3/4 [   3.830s] linkerd-policy-test::e2e_egress_network default_traffic_policy_http_allow
   FLAKY 2/4 [  37.004s] linkerd-policy-test::e2e_http_local_ratelimit_policy ratelimit_total
   FLAKY 2/4 [   7.947s] linkerd-policy-test::e2e_server_authorization network
   FLAKY 2/4 [   0.142s] linkerd-policy-test::inbound_http_route_status inbound_accepted_parent
   FLAKY 2/4 [   0.167s] linkerd-policy-test::inbound_http_route_status inbound_multiple_parents
   FLAKY 2/4 [   1.681s] linkerd-policy-test::outbound_api multiple_routes
   FLAKY 2/4 [   1.013s] linkerd-policy-test::outbound_api routes_without_backends
   FLAKY 3/4 [   1.153s] linkerd-policy-test::outbound_api service_with_routes_with_cross_namespace_backend
   FLAKY 2/4 [   0.282s] linkerd-policy-test::outbound_api_failure_accrual consecutive_failure_accrual
   FLAKY 3/4 [   0.290s] linkerd-policy-test::outbound_api_failure_accrual default_failure_accrual
   FLAKY 2/4 [   0.354s] linkerd-policy-test::outbound_api_http http_route_gateway_timeouts
   FLAKY 2/4 [   0.740s] linkerd-policy-test::outbound_api_http http_route_retries_and_timeouts
   ```

After having measured this, we also added a watcher for the namespace, vastly reducing flakiness:

```
     Summary [ 517.330s] 151 tests run: 151 passed (2 flaky), 0 skipped
   FLAKY 2/4 [   0.941s] linkerd-policy-test::outbound_api routes_without_backends
   FLAKY 2/4 [   0.459s] linkerd-policy-test::outbound_api_tcp multiple_tcp_routes
```

Finally, the jaeger chart version has to be pinned to an earlier one as the latest one is presenting some breaking changes that are making the tracing test fail.

Author: alpeb

policy-test/src/lib.rs

@@ -708,7 +708,7 @@ where
     .await;
     tracing::trace!(?ns);
     tokio::time::timeout(
-        tokio::time::Duration::from_secs(60),
+        tokio::time::Duration::from_secs(15),
         await_service_account(&client, &ns.name_unchecked(), "default"),
     )
     .await
@@ -754,33 +754,57 @@ where
     }
 }
 
-pub async fn await_service_account(client: &kube::Client, ns: &str, name: &str) {
+async fn await_resource<T>(
+    mut watcher: impl futures::Stream<
+            Item = Result<kube::runtime::watcher::Event<T>, kube::runtime::watcher::Error>,
+        > + Unpin,
+    predicate: impl Fn(&T) -> bool,
+) where
+    T: kube::Resource + std::fmt::Debug,
+{
     use futures::StreamExt;
 
-    tracing::trace!(%name, %ns, "Waiting for serviceaccount");
-    tokio::pin! {
-        let sas = kube::runtime::watcher(
-            kube::Api::<k8s::ServiceAccount>::namespaced(client.clone(), ns),
-            Default::default(),
-        );
-    }
     loop {
-        let ev = sas
+        let ev = watcher
             .next()
             .await
-            .expect("serviceaccounts watch must not end")
-            .expect("serviceaccounts watch must not fail");
+            .expect("watch must not end")
+            .expect("watch must not fail");
         tracing::info!(?ev);
         match ev {
-            kube::runtime::watcher::Event::InitApply(sa)
-            | kube::runtime::watcher::Event::Apply(sa)
-                if sa.name_unchecked() == name =>
+            kube::runtime::watcher::Event::InitApply(resource)
+            | kube::runtime::watcher::Event::Apply(resource)
+                if predicate(&resource) =>
             {
-                return
+                break
             }
             _ => {}
         }
     }
+}
+
+pub async fn await_service_account(client: &kube::Client, ns: &str, name: &str) {
+    tracing::trace!(%ns, "Waiting for namespace");
+
+    let label_selector = format!("kubernetes.io/metadata.name={}", ns);
+    let watcher_config = kube::runtime::watcher::Config::default().labels(&label_selector);
+    tokio::pin! {
+        let namespaces = kube::runtime::watcher(
+            kube::Api::<k8s::Namespace>::all(client.clone()),
+            watcher_config,
+        );
+    }
+    await_resource(namespaces, |namespace| namespace.name_unchecked() == ns).await;
+
+    tracing::trace!(%name, %ns, "Waiting for serviceaccount");
+
+    tokio::pin! {
+        let sas = kube::runtime::watcher(
+            kube::Api::<k8s::ServiceAccount>::namespaced(client.clone(), ns),
+            Default::default(),
+        );
+    }
+    await_resource(sas, |sa| sa.name_unchecked() == name).await;
 
     // XXX In some versions of k8s, it may be appropriate to wait for the
     // ServiceAccount's secret to be created, but, as of v1.24,

test/integration/tracing/tracing/tracing_test.go

@@ -134,7 +134,7 @@ func installTracing(t *testing.T, namespace string) {
 	if err != nil {
 		testutil.AnnotatedFatalf(t, "failed to add OpenTelemetry repository", "failed to add OpenTelemetry repository\n%s\n------\n%s\n", stdout, stderr)
 	}
-	stdout, stderr, err = TestHelper.HelmRun("install", "jaeger", "jaegertracing/jaeger", "--namespace=tracing", "--values=testdata/jaeger-aio-values.yaml")
+	stdout, stderr, err = TestHelper.HelmRun("install", "jaeger", "jaegertracing/jaeger", "--namespace=tracing", "--values=testdata/jaeger-aio-values.yaml", "--version=3.4.1")
 	if err != nil {
 		testutil.AnnotatedFatalf(t, "failed to install jaeger", "failed to install jaeger\n%s\n------\n%s\n", stdout, stderr)
 	}